1 from __future__
import absolute_import, division, print_function
10 from builtins
import zip
11 from builtins
import range
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
30 from .checksum
import checksum
31 from .utils
import getDataRef
35 """Parameters controlling the measurement of background statistics""" 36 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37 default=int(afwMath.MEANCLIP))
38 clip = Field(doc=
"Clipping threshold for background",
39 dtype=float, default=3.0)
40 nIter = Field(doc=
"Clipping iterations for background",
42 maxVisitsToCalcErrorFromInputVariance = Field(
43 doc=
"Maximum number of visits to estimate variance from input variance, not per-pixel spread",
45 mask = ListField(doc=
"Mask planes to reject",
46 dtype=str, default=[
"DETECTED",
"BAD",
"NO_DATA",])
50 """Measure statistics on the background 52 This can be useful for scaling the background, e.g., for flats and fringe frames. 54 ConfigClass = CalibStatsConfig
56 def run(self, exposureOrImage):
57 """!Measure a particular statistic on an image (of some sort). 59 @param exposureOrImage Exposure, MaskedImage or Image. 60 @return Value of desired statistic 62 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
63 afwImage.Mask.getPlaneBitMask(self.config.mask))
65 image = exposureOrImage.getMaskedImage()
68 image = exposureOrImage.getImage()
70 image = exposureOrImage
72 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
76 """Configuration for combining calib images""" 77 rows = Field(doc=
"Number of rows to read at a time",
78 dtype=int, default=512)
79 mask = ListField(doc=
"Mask planes to respect", dtype=str,
80 default=[
"SAT",
"DETECTED",
"INTRP"])
81 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
82 default=int(afwMath.MEANCLIP))
83 clip = Field(doc=
"Clipping threshold for combination",
84 dtype=float, default=3.0)
85 nIter = Field(doc=
"Clipping iterations for combination",
87 stats = ConfigurableField(target=CalibStatsTask,
88 doc=
"Background statistics configuration")
92 """Task to combine calib images""" 93 ConfigClass = CalibCombineConfig
96 Task.__init__(self, *args, **kwargs)
97 self.makeSubtask(
"stats")
99 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
100 """!Combine calib images for a single sensor 102 @param sensorRefList List of data references to combine (for a single sensor) 103 @param expScales List of scales to apply for each exposure 104 @param finalScale Desired scale for final combined image 105 @param inputName Data set name for inputs 106 @return combined image 109 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
110 afwImage.Mask.getPlaneBitMask(self.config.mask))
111 numImages = len(sensorRefList)
112 if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
113 stats.setCalcErrorFromInputVariance(
True)
116 combined = afwImage.MaskedImageF(width, height)
117 imageList = [
None]*numImages
118 for start
in range(0, height, self.config.rows):
119 rows = min(self.config.rows, height - start)
120 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
121 afwGeom.Extent2I(width, rows))
122 subCombined = combined.Factory(combined, box)
124 for i, sensorRef
in enumerate(sensorRefList):
125 if sensorRef
is None:
128 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
129 if expScales
is not None:
131 imageList[i] = exposure.getMaskedImage()
133 self.
combine(subCombined, imageList, stats)
135 if finalScale
is not None:
136 background = self.stats.
run(combined)
137 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
138 (NODE, background, finalScale))
139 combined *= finalScale / background
144 """Get dimensions of the inputs""" 146 for sensorRef
in sensorRefList:
147 if sensorRef
is None:
149 md = sensorRef.get(inputName +
"_md")
150 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
154 """Apply scale to input exposure 156 This implementation applies a flux scaling: the input exposure is 157 divided by the provided scale. 159 if scale
is not None:
160 mi = exposure.getMaskedImage()
164 """!Combine multiple images 166 @param target Target image to receive the combined pixels 167 @param imageList List of input images 168 @param stats Statistics control 170 images = [img
for img
in imageList
if img
is not None]
171 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
175 """Determine a consistent size, given a list of image sizes""" 176 dim = set((w, h)
for w, h
in dimList)
179 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
184 """!Return a tuple of specific values from a dict 186 This provides a hashable representation of the dict from certain keywords. 187 This can be useful for creating e.g., a tuple of the values in the DataId 188 that identify the CCD. 190 @param dict_ dict to parse 191 @param keys keys to extract (order is important) 192 @return tuple of values 194 return tuple(dict_[k]
for k
in keys)
198 """!Determine a list of CCDs from exposure references 200 This essentially inverts the exposure-level references (which 201 provides a list of CCDs for each exposure), by providing 202 a dataId list for each CCD. Consider an input list of exposures 203 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this 206 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]} 208 This is a dict whose keys are tuples of the identifying values of a 209 CCD (usually just the CCD number) and the values are lists of dataIds 210 for that CCD in each exposure. A missing dataId is given the value 213 @param expRefList List of data references for exposures 214 @param level Level for the butler to generate CCDs 215 @param ccdKeys DataId keywords that identify a CCD 216 @return dict of data identifier lists for each CCD 218 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
219 level)]
for expRef
in expRefList]
222 ccdKeys = set(ccdKeys)
224 for ccdIdList
in expIdList:
225 for ccdId
in ccdIdList:
232 for n, ccdIdList
in enumerate(expIdList):
233 for ccdId
in ccdIdList:
235 if name
not in ccdLists:
237 ccdLists[name].append(ccdId)
241 ccdLists[ccd] = sorted(ccdLists[ccd], key=
lambda dd:
dictToTuple(dd, sorted(dd.keys())))
247 """Generate a matrix of results using pool.map 249 The function should have the call signature: 250 func(cache, dataId, *args, **kwargs) 252 We return a dict mapping 'ccd name' to a list of values for 255 @param pool Process pool 256 @param func Function to call for each dataId 257 @param ccdIdLists Dict of data identifier lists for each CCD name 258 @return matrix of results 260 dataIdList = sum(ccdIdLists.values(), [])
261 resultList = pool.map(func, dataIdList, *args, **kwargs)
263 data = dict((ccdName, [
None] * len(expList))
for ccdName, expList
in ccdIdLists.items())
264 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
265 for expNum, dataId
in enumerate(expList)]
266 for ccdName, expList
in ccdIdLists.items()], []))
267 for dataId, result
in zip(dataIdList, resultList):
270 ccdName, expNum = indices[tuple(dataId.values())]
271 data[ccdName][expNum] = result
276 """Split name=value pairs and put the result in a dict""" 278 def __call__(self, parser, namespace, values, option_string):
279 output = getattr(namespace, self.dest, {})
280 for nameValue
in values:
281 name, sep, valueStr = nameValue.partition(
"=")
283 parser.error(
"%s value %s must be in form name=value" %
284 (option_string, nameValue))
285 output[name] = valueStr
286 setattr(namespace, self.dest, output)
290 """ArgumentParser for calibration construction""" 293 """Add a --calibId argument to the standard pipe_base argument parser""" 294 ArgumentParser.__init__(self, *args, **kwargs)
296 self.add_id_argument(
"--id", datasetType=
"raw",
297 help=
"input identifiers, e.g., --id visit=123 ccd=4")
298 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
299 help=
"identifiers for calib, e.g., --calibId version=1",
300 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
305 Checks that the "--calibId" provided works. 307 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
309 keys = namespace.butler.getKeys(self.
calibName)
311 for name, value
in namespace.calibId.items():
314 "%s is not a relevant calib identifier key (%s)" % (name, keys))
315 parsed[name] = keys[name](value)
316 namespace.calibId = parsed
322 """Configuration for constructing calibs""" 323 clobber = Field(dtype=bool, default=
True,
324 doc=
"Clobber existing processed images?")
325 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
326 dateObs = Field(dtype=str, default=
"dateObs",
327 doc=
"Key for observation date in exposure registry")
328 dateCalib = Field(dtype=str, default=
"calibDate",
329 doc=
"Key for calib date in calib registry")
330 filter = Field(dtype=str, default=
"filter",
331 doc=
"Key for filter name in exposure/calib registries")
332 combination = ConfigurableField(
333 target=CalibCombineTask, doc=
"Calib combination configuration")
334 ccdKeys = ListField(dtype=str, default=[
"ccd"],
335 doc=
"DataId keywords specifying a CCD")
336 visitKeys = ListField(dtype=str, default=[
"visit"],
337 doc=
"DataId keywords specifying a visit")
338 calibKeys = ListField(dtype=str, default=[],
339 doc=
"DataId keywords specifying a calibration")
340 doCameraImage = Field(dtype=bool, default=
True, doc=
"Create camera overview image?")
341 binning = Field(dtype=int, default=64, doc=
"Binning to apply for camera image")
344 self.
isr.doWrite =
False 348 """Get parsed values into the CalibTask.run""" 351 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
354 """Call the Task with the kwargs from getTargetList""" 355 task = self.TaskClass(config=self.config, log=self.log)
358 result = task.run(**args)
361 result = task.run(**args)
362 except Exception
as e:
365 task.log.fatal(
"Failed: %s" % e)
366 traceback.print_exc(file=sys.stderr)
368 if self.doReturnResults:
370 exitStatus=exitStatus,
372 metadata=task.metadata,
377 exitStatus=exitStatus,
381 """!Base class for constructing calibs. 383 This should be subclassed for each of the required calib types. 384 The subclass should be sure to define the following class variables: 385 * _DefaultName: default name of the task, used by CmdLineTask 386 * calibName: name of the calibration data set in the butler 387 The subclass may optionally set: 388 * filterName: filter name to give the resultant calib 390 ConfigClass = CalibConfig
391 RunnerClass = CalibTaskRunner
398 BatchPoolTask.__init__(self, *args, **kwargs)
399 self.makeSubtask(
"isr")
400 self.makeSubtask(
"combination")
404 numCcds = len(parsedCmd.butler.get(
"camera"))
406 parsedCmd)[0][
'expRefList'])
407 numCycles = int(numCcds/float(numCores) + 0.5)
408 return time*numExps*numCycles
411 def _makeArgumentParser(cls, *args, **kwargs):
412 kwargs.pop(
"doBatch",
False)
415 def run(self, expRefList, butler, calibId):
416 """!Construct a calib from a list of exposure references 418 This is the entry point, called by the TaskRunner.__call__ 420 Only the master node executes this method. 422 @param expRefList List of data references at the exposure level 423 @param butler Data butler 424 @param calibId Identifier dict for calib 426 for expRef
in expRefList:
427 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
431 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
434 outputIdItemList = list(outputId.items())
435 for ccdName
in ccdIdLists:
436 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
437 dataId.update(outputIdItemList)
439 dataId.update(outputIdItemList)
442 butler.get(self.
calibName +
"_filename", dataId)
443 except Exception
as e:
445 "Unable to determine output filename \"%s_filename\" from %s: %s" %
448 processPool =
Pool(
"process")
449 processPool.storeSet(butler=butler)
455 scales = self.
scale(ccdIdLists, data)
457 combinePool =
Pool(
"combine")
458 combinePool.storeSet(butler=butler)
461 calibs = self.
scatterCombine(combinePool, outputId, ccdIdLists, scales)
463 if self.config.doCameraImage:
464 camera = butler.get(
"camera")
468 butler.put(cameraImage, self.
calibName +
"_camera", dataId)
469 except Exception
as exc:
470 self.log.warn(
"Unable to create camera image: %s" % (exc,))
474 ccdIdLists = ccdIdLists,
477 processPool = processPool,
478 combinePool = combinePool,
482 """!Generate the data identifier for the output calib 484 The mean date and the common filter are included, using keywords 485 from the configuration. The CCD-specific part is not included 486 in the data identifier. 488 @param expRefList List of data references at exposure level 489 @param calibId Data identifier elements for the calib provided by the user 490 @return data identifier 494 for expRef
in expRefList:
495 butler = expRef.getButler()
496 dataId = expRef.dataId
498 midTime += self.
getMjd(butler, dataId)
501 if filterName
is None:
502 filterName = thisFilter
503 elif filterName != thisFilter:
504 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
505 dataId, thisFilter, filterName))
507 midTime /= len(expRefList)
508 date = str(dafBase.DateTime(
509 midTime, dafBase.DateTime.MJD).toPython().date())
511 outputId = {self.config.filter: filterName,
512 self.config.dateCalib: date}
513 outputId.update(calibId)
516 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
517 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier""" 518 if self.config.dateObs
in dataId:
519 dateObs = dataId[self.config.dateObs]
521 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
522 if "T" not in dateObs:
523 dateObs = dateObs +
"T12:00:00.0Z" 524 elif not dateObs.endswith(
"Z"):
527 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
530 """Determine the filter from a data identifier""" 531 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
535 if calibName
is None:
538 if missingKeys
is None:
539 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
541 for k
in missingKeys:
543 v = butler.queryMetadata(
'raw', [k], dataId)
544 except Exception
as e:
553 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
556 """!Update the metadata from the VisitInfo 558 \param calibImage The image whose metadata is to be set 559 \param exposureTime The exposure time for the image 560 \param darkTime The time since the last read (default: exposureTime) 564 darkTime = exposureTime
566 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
567 md = calibImage.getMetadata()
569 afwImage.setVisitInfoMetadata(md, visitInfo)
572 """!Scatter the processing among the nodes 574 We scatter each CCD independently (exposures aren't grouped together), 575 to make full use of all available processors. This necessitates piecing 576 everything back together in the same format as ccdIdLists afterwards. 578 Only the master node executes this method. 580 @param pool Process pool 581 @param ccdIdLists Dict of data identifier lists for each CCD name 582 @return Dict of lists of returned data for each CCD name 584 self.log.info(
"Scatter processing")
587 def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
588 """!Process a CCD, specified by a data identifier 590 After processing, optionally returns a result (produced by 591 the 'processResult' method) calculated from the processed 592 exposure. These results will be gathered by the master node, 593 and is a means for coordinated scaling of all CCDs for flats, 596 Only slave nodes execute this method. 598 @param cache Process pool cache 599 @param ccdId Data identifier for CCD 600 @param outputName Output dataset name for butler 601 @return result from 'processResult' 604 self.log.warn(
"Null identifier received on %s" % NODE)
607 if self.config.clobber
or not sensorRef.datasetExists(outputName):
608 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
611 except Exception
as e:
612 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
618 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
619 exposure = sensorRef.get(outputName)
623 """Process a single CCD, specified by a data reference 625 Generally, this simply means doing ISR. 627 Only slave nodes execute this method. 629 return self.isr.runDataRef(dataRef).exposure
632 """!Write the processed CCD 634 We need to write these out because we can't hold them all in 637 Only slave nodes execute this method. 639 @param dataRef Data reference 640 @param exposure CCD exposure to write 641 @param outputName Output dataset name for butler. 643 dataRef.put(exposure, outputName)
646 """Extract processing results from a processed exposure 648 This method generates what is gathered by the master node. 649 This can be a background measurement or similar for scaling 650 flat-fields. It must be picklable! 652 Only slave nodes execute this method. 657 """!Determine scaling across CCDs and exposures 659 This is necessary mainly for flats, so as to determine a 660 consistent scaling across the entire focal plane. This 661 implementation is simply a placeholder. 663 Only the master node executes this method. 665 @param ccdIdLists Dict of data identifier lists for each CCD tuple 666 @param data Dict of lists of returned data for each CCD tuple 667 @return dict of Struct(ccdScale: scaling for CCD, 668 expScales: scaling for each exposure 671 self.log.info(
"Scale on %s" % NODE)
672 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
673 for name
in ccdIdLists)
676 """!Scatter the combination of exposures across multiple nodes 678 In this case, we can only scatter across as many nodes as 681 Only the master node executes this method. 683 @param pool Process pool 684 @param outputId Output identifier (exposure part only) 685 @param ccdIdLists Dict of data identifier lists for each CCD name 686 @param scales Dict of structs with scales, for each CCD name 687 @param dict of binned images 689 self.log.info(
"Scatter combination")
690 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for 691 ccdName
in ccdIdLists]
692 images = pool.map(self.
combine, data, outputId)
693 return dict(zip(ccdIdLists.keys(), images))
696 """Get fully-qualified output data identifier 698 We may need to look up keys that aren't in the output dataId. 700 @param ccdName Name tuple for CCD 701 @param butler Data butler 702 @param outputId Data identifier for combined image (exposure part only) 703 @return fully-qualified output dataId 705 fullOutputId = {k: ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
706 fullOutputId.update(outputId)
708 fullOutputId.update(outputId)
712 """!Combine multiple exposures of a particular CCD and write the output 714 Only the slave nodes execute this method. 716 @param cache Process pool cache 717 @param struct Parameters for the combination, which has the following components: 718 * ccdName Name tuple for CCD 719 * ccdIdList List of data identifiers for combination 720 * scales Scales to apply (expScales are scalings for each exposure, 721 ccdScale is final scale for combined image) 722 @param outputId Data identifier for combined image (exposure part only) 723 @return binned calib image 726 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 727 dataId
in struct.ccdIdList]
728 self.log.info(
"Combining %s on %s" % (outputId, NODE))
729 calib = self.combination.
run(dataRefList, expScales=struct.scales.expScales,
730 finalScale=struct.scales.ccdScale)
732 if not hasattr(calib,
"getMetadata"):
733 if hasattr(calib,
"getVariance"):
734 calib = afwImage.makeExposure(calib)
736 calib = afwImage.DecoratedImageF(calib.getImage())
741 struct.ccdIdList, outputId)
745 self.
write(cache.butler, calib, outputId)
747 return afwMath.binImage(calib.getImage(), self.config.binning)
750 """!Record metadata including the inputs and creation details 752 This metadata will go into the FITS header. 754 @param butler Data butler 755 @param calib Combined calib exposure. 756 @param dataIdList List of data identifiers for calibration inputs 757 @param outputId Data identifier for output 759 header = calib.getMetadata()
763 now = time.localtime()
764 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
765 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
767 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
770 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if 772 for i, v
in enumerate(sorted(set(visits))):
773 header.add(
"CALIB_INPUT_%d" % (i,), v)
775 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
776 for key, value
in outputId.items()))
780 """Interpolate over NANs in the combined image 782 NANs can result from masked areas on the CCD. We don't want them getting 783 into our science images, so we replace them with the median of the image. 785 if hasattr(image,
"getMaskedImage"):
787 image = image.getMaskedImage().getImage()
788 if hasattr(image,
"getImage"):
789 image = image.getImage()
790 array = image.getArray()
791 bad = np.isnan(array)
792 array[bad] = np.median(array[np.logical_not(bad)])
794 def write(self, butler, exposure, dataId):
795 """!Write the final combined calib 797 Only the slave nodes execute this method 799 @param butler Data butler 800 @param exposure CCD exposure to write 801 @param dataId Data identifier for output 803 self.log.info(
"Writing %s on %s" % (dataId, NODE))
804 butler.put(exposure, self.
calibName, dataId)
807 """!Create and write an image of the entire camera 809 This is useful for judging the quality or getting an overview of 810 the features of the calib. 812 This requires that the 'ccd name' is a tuple containing only the 813 detector ID. If that is not the case, change CalibConfig.ccdKeys 814 or set CalibConfig.doCameraImage=False to disable this. 816 @param camera Camera object 817 @param dataId Data identifier for output 818 @param calibs Dict mapping 'ccd name' to calib image 821 class ImageSource(object):
822 """Source of images for makeImageFromCamera 824 This assumes that the 'ccd name' is a tuple containing 825 only the detector ID. 832 def getCcdImage(self, detector, imageFactory, binSize):
833 detId = (detector.getId(),)
834 if detId
not in self.
images:
835 return imageFactory(1, 1), detId
836 return self.
images[detId], detId
838 image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
839 binSize=self.config.binning)
843 """Configuration for bias construction. 845 No changes required compared to the base class, but 846 subclassed for distinction. 851 class BiasTask(CalibTask):
852 """Bias construction""" 853 ConfigClass = BiasConfig
854 _DefaultName =
"bias" 861 """Overrides to apply for bias construction""" 862 config.isr.doBias =
False 863 config.isr.doDark =
False 864 config.isr.doFlat =
False 865 config.isr.doFringe =
False 869 """Configuration for dark construction""" 870 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
871 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
872 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
873 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
874 repair = ConfigurableField(
875 target=RepairTask, doc=
"Task to repair artifacts")
878 CalibConfig.setDefaults(self)
885 The only major difference from the base class is a cosmic-ray 886 identification stage, and dividing each image by the dark time 887 to generate images of the dark rate. 889 ConfigClass = DarkConfig
890 _DefaultName =
"dark" 895 CalibTask.__init__(self, *args, **kwargs)
896 self.makeSubtask(
"repair")
900 """Overrides to apply for dark construction""" 901 config.isr.doDark =
False 902 config.isr.doFlat =
False 903 config.isr.doFringe =
False 906 """Process a single CCD 908 Besides the regular ISR, also masks cosmic-rays and divides each 909 processed image by the dark time to generate images of the dark rate. 910 The dark time is provided by the 'getDarkTime' method. 912 exposure = CalibTask.processSingle(self, sensorRef)
914 if self.config.doRepair:
915 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
916 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
918 self.repair.
run(exposure, keepCRs=
False)
919 if self.config.crGrow > 0:
920 mask = exposure.getMaskedImage().getMask().clone()
921 mask &= mask.getPlaneBitMask(
"CR")
922 fpSet = afwDet.FootprintSet(
923 mask, afwDet.Threshold(0.5))
924 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
925 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
927 mi = exposure.getMaskedImage()
932 """Retrieve the dark time for an exposure""" 933 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
934 if not np.isfinite(darkTime):
935 raise RuntimeError(
"Non-finite darkTime")
940 """Configuration for flat construction""" 941 iterations = Field(dtype=int, default=10,
942 doc=
"Number of iterations for scale determination")
943 stats = ConfigurableField(target=CalibStatsTask,
944 doc=
"Background statistics configuration")
950 The principal change from the base class involves gathering the background 951 values from each image and using them to determine the scalings for the final 954 ConfigClass = FlatConfig
955 _DefaultName =
"flat" 960 """Overrides for flat construction""" 961 config.isr.doFlat =
False 962 config.isr.doFringe =
False 965 CalibTask.__init__(self, *args, **kwargs)
966 self.makeSubtask(
"stats")
969 return self.stats.
run(exposure)
972 """Determine the scalings for the final combination 974 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling 975 of one CCD to all the others in an exposure, and E_j is the scaling 976 of the exposure. We convert everything to logarithms so we can work 977 with a linear system. We determine the C_i and E_j from B_ij by iteration, 978 under the additional constraint that the average CCD scale is unity. 980 This algorithm comes from Eugene Magnier and Pan-STARRS. 982 assert len(ccdIdLists.values()) > 0,
"No successful CCDs" 983 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
985 lengths) == 1,
"Number of successful exposures for each CCD differs" 986 assert tuple(lengths)[0] > 0,
"No successful exposures" 988 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
989 bgMatrix = np.array([[0.0] * len(expList)
990 for expList
in ccdIdLists.values()])
991 for name
in ccdIdLists:
994 d
if d
is not None else np.nan
for d
in data[name]]
996 numpyPrint = np.get_printoptions()
997 np.set_printoptions(threshold=np.inf)
998 self.log.info(
"Input backgrounds: %s" % bgMatrix)
1001 numCcds = len(ccdIdLists)
1002 numExps = bgMatrix.shape[1]
1004 bgMatrix = np.log(bgMatrix)
1005 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1007 compScales = np.zeros(numCcds)
1008 expScales = np.array(
1009 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
1011 for iterate
in range(self.config.iterations):
1012 compScales = np.array(
1013 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
1014 expScales = np.array(
1015 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
1017 avgScale = np.average(np.exp(compScales))
1018 compScales -= np.log(avgScale)
1019 self.log.debug(
"Iteration %d exposure scales: %s",
1020 iterate, np.exp(expScales))
1021 self.log.debug(
"Iteration %d component scales: %s",
1022 iterate, np.exp(compScales))
1024 expScales = np.array(
1025 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
1027 if np.any(np.isnan(expScales)):
1028 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
1029 (bgMatrix, expScales))
1031 expScales = np.exp(expScales)
1032 compScales = np.exp(compScales)
1034 self.log.info(
"Exposure scales: %s" % expScales)
1035 self.log.info(
"Component relative scaling: %s" % compScales)
1036 np.set_printoptions(**numpyPrint)
1038 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1039 for ccdName
in ccdIdLists)
1043 """Configuration for fringe construction""" 1044 stats = ConfigurableField(target=CalibStatsTask,
1045 doc=
"Background statistics configuration")
1046 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1047 doc=
"Background configuration")
1048 detection = ConfigurableField(
1049 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1050 detectSigma = Field(dtype=float, default=1.0,
1051 doc=
"Detection PSF gaussian sigma")
1055 """Fringe construction task 1057 The principal change from the base class is that the images are 1058 background-subtracted and rescaled by the background. 1060 XXX This is probably not right for a straight-up combination, as we 1061 are currently doing, since the fringe amplitudes need not scale with 1064 XXX Would like to have this do PCA and generate multiple images, but 1065 that will take a bit of work with the persistence code. 1067 ConfigClass = FringeConfig
1068 _DefaultName =
"fringe" 1069 calibName =
"fringe" 1073 """Overrides for fringe construction""" 1074 config.isr.doFringe =
False 1077 CalibTask.__init__(self, *args, **kwargs)
1078 self.makeSubtask(
"detection")
1079 self.makeSubtask(
"stats")
1080 self.makeSubtask(
"subtractBackground")
1083 """Subtract the background and normalise by the background level""" 1084 exposure = CalibTask.processSingle(self, sensorRef)
1085 bgLevel = self.stats.
run(exposure)
1086 self.subtractBackground.
run(exposure)
1087 mi = exposure.getMaskedImage()
1089 footprintSets = self.detection.detectFootprints(
1090 exposure, sigma=self.config.detectSigma)
1091 mask = exposure.getMaskedImage().getMask()
1092 detected = 1 << mask.addMaskPlane(
"DETECTED")
1093 for fpSet
in (footprintSets.positive, footprintSets.negative):
1094 if fpSet
is not None:
1095 afwDet.setMaskFromFootprintList(
1096 mask, fpSet.getFootprints(), detected)
1101 """Configuration for sky frame construction""" 1102 detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1103 detectSigma = Field(dtype=float, default=2.0, doc=
"Detection PSF gaussian sigma")
1104 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1105 doc=
"Regular-scale background configuration, for object detection")
1106 largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1107 doc=
"Large-scale background configuration")
1108 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
1109 maskThresh = Field(dtype=float, default=3.0, doc=
"k-sigma threshold for masking pixels")
1110 mask = ListField(dtype=str, default=[
"BAD",
"SAT",
"DETECTED",
"NO_DATA"],
1111 doc=
"Mask planes to consider as contaminated")
1115 """Task for sky frame construction 1117 The sky frame is a (relatively) small-scale background 1118 model, the response of the camera to the sky. 1120 To construct, we first remove a large-scale background (e.g., caused 1121 by moonlight) which may vary from image to image. Then we construct a 1122 model of the sky, which is essentially a binned version of the image 1123 (important configuration parameters: sky.background.[xy]BinSize). 1124 It is these models which are coadded to yield the sky frame. 1126 ConfigClass = SkyConfig
1127 _DefaultName =
"sky" 1131 CalibTask.__init__(self, *args, **kwargs)
1132 self.makeSubtask(
"detection")
1133 self.makeSubtask(
"subtractBackground")
1134 self.makeSubtask(
"sky")
1137 """!Scatter the processing among the nodes 1139 Only the master node executes this method, assigning work to the 1142 We measure and subtract off a large-scale background model across 1143 all CCDs, which requires a scatter/gather. Then we process the 1144 individual CCDs, subtracting the large-scale background model and 1145 the residual background model measured. These residuals will be 1146 combined for the sky frame. 1148 @param pool Process pool 1149 @param ccdIdLists Dict of data identifier lists for each CCD name 1150 @return Dict of lists of returned data for each CCD name 1152 self.log.info(
"Scatter processing")
1154 numExps = set(len(expList)
for expList
in ccdIdLists.values())
1155 assert len(numExps) == 1
1156 numExps = numExps.pop()
1164 for exp
in range(numExps):
1165 bgModels = [bgModelList[ccdName][exp]
for ccdName
in ccdIdLists]
1166 visit = set(tuple(ccdIdLists[ccdName][exp][key]
for key
in sorted(self.config.visitKeys))
for 1167 ccdName
in ccdIdLists)
1168 assert len(visit) == 1
1170 bgModel = bgModels[0]
1171 for bg
in bgModels[1:]:
1173 self.log.info(
"Background model min/max for visit %s: %f %f", visit,
1174 np.min(bgModel.getStatsImage().getArray()),
1175 np.max(bgModel.getStatsImage().getArray()))
1176 backgrounds[visit] = bgModel
1177 scales[visit] = np.median(bgModel.getStatsImage().getArray())
1179 return mapToMatrix(pool, self.
process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1182 """!Measure background model for CCD 1184 This method is executed by the slaves. 1186 The background models for all CCDs in an exposure will be 1187 combined to form a full focal-plane background model. 1189 @param cache Process pool cache 1190 @param dataId Data identifier 1191 @return Bcakground model 1198 config = self.config.largeScaleBackground
1199 camera = dataRef.get(
"camera")
1200 bgModel = FocalPlaneBackground.fromCamera(config, camera)
1201 bgModel.addCcd(exposure)
1205 """!Process a single CCD for the background 1207 This method is executed by the slaves. 1209 Because we're interested in the background, we detect and mask astrophysical 1210 sources, and pixels above the noise level. 1212 @param dataRef Data reference for CCD. 1213 @return processed exposure 1215 if not self.config.clobber
and dataRef.datasetExists(
"postISRCCD"):
1216 return dataRef.get(
"postISRCCD")
1217 exposure = CalibTask.processSingle(self, dataRef)
1220 bgTemp = self.subtractBackground.
run(exposure).background
1221 footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1222 image = exposure.getMaskedImage()
1223 if footprints.background
is not None:
1224 image += footprints.background.getImageF()
1227 variance = image.getVariance()
1228 noise = np.sqrt(np.median(variance.getArray()))
1229 isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1230 image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask(
"DETECTED")
1233 image += bgTemp.getImage()
1236 maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1237 isBad = image.getMask().getArray() & maskVal > 0
1238 bgLevel = np.median(image.getImage().getArray()[~isBad])
1239 image.getImage().getArray()[isBad] = bgLevel
1240 dataRef.put(exposure,
"postISRCCD")
1244 """Process a single CCD, specified by a data reference 1246 We subtract the appropriate focal plane background model, 1247 divide by the appropriate scale and measure the background. 1249 Only slave nodes execute this method. 1251 @param dataRef Data reference for single CCD 1252 @param backgrounds Background model for each visit 1253 @param scales Scales for each visit 1254 @return Processed exposure 1256 visit = tuple(dataRef.dataId[key]
for key
in sorted(self.config.visitKeys))
1257 exposure = dataRef.get(
"postISRCCD", immediate=
True)
1258 image = exposure.getMaskedImage()
1259 detector = exposure.getDetector()
1260 bbox = image.getBBox()
1262 bgModel = backgrounds[visit]
1263 bg = bgModel.toCcdBackground(detector, bbox)
1264 image -= bg.getImage()
1265 image /= scales[visit]
1268 dataRef.put(bg,
"icExpBackground")
1272 """!Combine multiple background models of a particular CCD and write the output 1274 Only the slave nodes execute this method. 1276 @param cache Process pool cache 1277 @param struct Parameters for the combination, which has the following components: 1278 * ccdName Name tuple for CCD 1279 * ccdIdList List of data identifiers for combination 1280 @param outputId Data identifier for combined image (exposure part only) 1281 @return binned calib image 1284 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 1285 dataId
in struct.ccdIdList]
1286 self.log.info(
"Combining %s on %s" % (outputId, NODE))
1287 bgList = [dataRef.get(
"icExpBackground", immediate=
True).clone()
for dataRef
in dataRefList]
1289 bgExp = self.sky.averageBackgrounds(bgList)
1292 cache.butler.put(bgExp,
"sky", outputId)
1293 return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def applyOverrides(cls, config)
def getFilter(self, butler, dataId)
def __init__(self, args, kwargs)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def __init__(self, args, kwargs)
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def processResult(self, exposure)
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def __init__(self, calibName, args, kwargs)
def applyScale(self, exposure, scale=None)
def processSingle(self, sensorRef)
def applyOverrides(cls, config)
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def processSingle(self, sensorRef)
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def processSingle(self, dataRef)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def interpolateNans(self, image)
def getDataRef(butler, dataId, datasetType="raw")
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def scale(self, ccdIdLists, data)
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def parse_args(self, args, kwargs)
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def getDarkTime(self, exposure)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def __init__(self, args, kwargs)
def __init__(self, args, kwargs)
def applyOverrides(cls, config)
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def processResult(self, exposure)
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def __init__(self, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
def getTargetList(parsedCmd, kwargs)
def __init__(self, args, kwargs)
Base class for constructing calibs.
def applyOverrides(cls, config)
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)