1 from __future__
import absolute_import, division, print_function
10 from builtins
import zip
11 from builtins
import range
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
30 from .checksum
import checksum
31 from .utils
import getDataRef
35 """Parameters controlling the measurement of background statistics""" 36 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37 default=int(afwMath.MEANCLIP))
38 clip = Field(doc=
"Clipping threshold for background",
39 dtype=float, default=3.0)
40 nIter = Field(doc=
"Clipping iterations for background",
42 maxVisitsToCalcErrorFromInputVariance = Field(
43 doc=
"Maximum number of visits to estimate variance from input variance, not per-pixel spread",
45 mask = ListField(doc=
"Mask planes to reject",
46 dtype=str, default=[
"DETECTED",
"BAD",
"NO_DATA",])
50 """Measure statistics on the background 52 This can be useful for scaling the background, e.g., for flats and fringe frames. 54 ConfigClass = CalibStatsConfig
56 def run(self, exposureOrImage):
57 """!Measure a particular statistic on an image (of some sort). 59 @param exposureOrImage Exposure, MaskedImage or Image. 60 @return Value of desired statistic 62 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
63 afwImage.Mask.getPlaneBitMask(self.config.mask))
65 image = exposureOrImage.getMaskedImage()
68 image = exposureOrImage.getImage()
70 image = exposureOrImage
72 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
76 """Configuration for combining calib images""" 77 rows = Field(doc=
"Number of rows to read at a time",
78 dtype=int, default=512)
79 mask = ListField(doc=
"Mask planes to respect", dtype=str,
80 default=[
"SAT",
"DETECTED",
"INTRP"])
81 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
82 default=int(afwMath.MEANCLIP))
83 clip = Field(doc=
"Clipping threshold for combination",
84 dtype=float, default=3.0)
85 nIter = Field(doc=
"Clipping iterations for combination",
87 stats = ConfigurableField(target=CalibStatsTask,
88 doc=
"Background statistics configuration")
92 """Task to combine calib images""" 93 ConfigClass = CalibCombineConfig
96 Task.__init__(self, *args, **kwargs)
97 self.makeSubtask(
"stats")
99 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
100 """!Combine calib images for a single sensor 102 @param sensorRefList List of data references to combine (for a single sensor) 103 @param expScales List of scales to apply for each exposure 104 @param finalScale Desired scale for final combined image 105 @param inputName Data set name for inputs 106 @return combined image 109 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
110 afwImage.Mask.getPlaneBitMask(self.config.mask))
111 numImages = len(sensorRefList)
112 if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
113 stats.setCalcErrorFromInputVariance(
True)
116 combined = afwImage.MaskedImageF(width, height)
117 imageList = [
None]*numImages
118 for start
in range(0, height, self.config.rows):
119 rows = min(self.config.rows, height - start)
120 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
121 afwGeom.Extent2I(width, rows))
122 subCombined = combined.Factory(combined, box)
124 for i, sensorRef
in enumerate(sensorRefList):
125 if sensorRef
is None:
128 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
129 if expScales
is not None:
131 imageList[i] = exposure.getMaskedImage()
133 self.
combine(subCombined, imageList, stats)
135 if finalScale
is not None:
136 background = self.stats.
run(combined)
137 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
138 (NODE, background, finalScale))
139 combined *= finalScale / background
144 """Get dimensions of the inputs""" 146 for sensorRef
in sensorRefList:
147 if sensorRef
is None:
149 md = sensorRef.get(inputName +
"_md")
150 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
154 """Apply scale to input exposure 156 This implementation applies a flux scaling: the input exposure is 157 divided by the provided scale. 159 if scale
is not None:
160 mi = exposure.getMaskedImage()
164 """!Combine multiple images 166 @param target Target image to receive the combined pixels 167 @param imageList List of input images 168 @param stats Statistics control 170 images = [img
for img
in imageList
if img
is not None]
171 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
175 """Determine a consistent size, given a list of image sizes""" 176 dim = set((w, h)
for w, h
in dimList)
179 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
184 """!Return a tuple of specific values from a dict 186 This provides a hashable representation of the dict from certain keywords. 187 This can be useful for creating e.g., a tuple of the values in the DataId 188 that identify the CCD. 190 @param dict_ dict to parse 191 @param keys keys to extract (order is important) 192 @return tuple of values 194 return tuple(dict_[k]
for k
in keys)
198 """!Determine a list of CCDs from exposure references 200 This essentially inverts the exposure-level references (which 201 provides a list of CCDs for each exposure), by providing 202 a dataId list for each CCD. Consider an input list of exposures 203 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this 206 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]} 208 This is a dict whose keys are tuples of the identifying values of a 209 CCD (usually just the CCD number) and the values are lists of dataIds 210 for that CCD in each exposure. A missing dataId is given the value 213 @param expRefList List of data references for exposures 214 @param level Level for the butler to generate CCDs 215 @param ccdKeys DataId keywords that identify a CCD 216 @return dict of data identifier lists for each CCD; 217 keys are values of ccdKeys in order 219 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
220 level)]
for expRef
in expRefList]
223 if len(ccdKeys) != len(set(ccdKeys)):
224 raise RuntimeError(
"Duplicate keys found in ccdKeys: %s" % ccdKeys)
226 for ccdIdList
in expIdList:
227 for ccdId
in ccdIdList:
234 for n, ccdIdList
in enumerate(expIdList):
235 for ccdId
in ccdIdList:
237 if name
not in ccdLists:
239 ccdLists[name].append(ccdId)
243 ccdLists[ccd] = sorted(ccdLists[ccd], key=
lambda dd:
dictToTuple(dd, sorted(dd.keys())))
249 """Generate a matrix of results using pool.map 251 The function should have the call signature: 252 func(cache, dataId, *args, **kwargs) 254 We return a dict mapping 'ccd name' to a list of values for 257 @param pool Process pool 258 @param func Function to call for each dataId 259 @param ccdIdLists Dict of data identifier lists for each CCD name 260 @return matrix of results 262 dataIdList = sum(ccdIdLists.values(), [])
263 resultList = pool.map(func, dataIdList, *args, **kwargs)
265 data = dict((ccdName, [
None] * len(expList))
for ccdName, expList
in ccdIdLists.items())
266 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
267 for expNum, dataId
in enumerate(expList)]
268 for ccdName, expList
in ccdIdLists.items()], []))
269 for dataId, result
in zip(dataIdList, resultList):
272 ccdName, expNum = indices[tuple(dataId.values())]
273 data[ccdName][expNum] = result
278 """Split name=value pairs and put the result in a dict""" 280 def __call__(self, parser, namespace, values, option_string):
281 output = getattr(namespace, self.dest, {})
282 for nameValue
in values:
283 name, sep, valueStr = nameValue.partition(
"=")
285 parser.error(
"%s value %s must be in form name=value" %
286 (option_string, nameValue))
287 output[name] = valueStr
288 setattr(namespace, self.dest, output)
292 """ArgumentParser for calibration construction""" 295 """Add a --calibId argument to the standard pipe_base argument parser""" 296 ArgumentParser.__init__(self, *args, **kwargs)
298 self.add_id_argument(
"--id", datasetType=
"raw",
299 help=
"input identifiers, e.g., --id visit=123 ccd=4")
300 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
301 help=
"identifiers for calib, e.g., --calibId version=1",
302 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
307 Checks that the "--calibId" provided works. 309 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
311 keys = namespace.butler.getKeys(self.
calibName)
313 for name, value
in namespace.calibId.items():
316 "%s is not a relevant calib identifier key (%s)" % (name, keys))
317 parsed[name] = keys[name](value)
318 namespace.calibId = parsed
324 """Configuration for constructing calibs""" 325 clobber = Field(dtype=bool, default=
True,
326 doc=
"Clobber existing processed images?")
327 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
328 dateObs = Field(dtype=str, default=
"dateObs",
329 doc=
"Key for observation date in exposure registry")
330 dateCalib = Field(dtype=str, default=
"calibDate",
331 doc=
"Key for calib date in calib registry")
332 filter = Field(dtype=str, default=
"filter",
333 doc=
"Key for filter name in exposure/calib registries")
334 combination = ConfigurableField(
335 target=CalibCombineTask, doc=
"Calib combination configuration")
336 ccdKeys = ListField(dtype=str, default=[
"ccd"],
337 doc=
"DataId keywords specifying a CCD")
338 visitKeys = ListField(dtype=str, default=[
"visit"],
339 doc=
"DataId keywords specifying a visit")
340 calibKeys = ListField(dtype=str, default=[],
341 doc=
"DataId keywords specifying a calibration")
342 doCameraImage = Field(dtype=bool, default=
True, doc=
"Create camera overview image?")
343 binning = Field(dtype=int, default=64, doc=
"Binning to apply for camera image")
346 self.
isr.doWrite =
False 350 """Get parsed values into the CalibTask.run""" 353 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
356 """Call the Task with the kwargs from getTargetList""" 357 task = self.TaskClass(config=self.config, log=self.log)
360 result = task.run(**args)
363 result = task.run(**args)
364 except Exception
as e:
367 task.log.fatal(
"Failed: %s" % e)
368 traceback.print_exc(file=sys.stderr)
370 if self.doReturnResults:
372 exitStatus=exitStatus,
374 metadata=task.metadata,
379 exitStatus=exitStatus,
383 """!Base class for constructing calibs. 385 This should be subclassed for each of the required calib types. 386 The subclass should be sure to define the following class variables: 387 * _DefaultName: default name of the task, used by CmdLineTask 388 * calibName: name of the calibration data set in the butler 389 The subclass may optionally set: 390 * filterName: filter name to give the resultant calib 392 ConfigClass = CalibConfig
393 RunnerClass = CalibTaskRunner
400 BatchPoolTask.__init__(self, *args, **kwargs)
401 self.makeSubtask(
"isr")
402 self.makeSubtask(
"combination")
406 numCcds = len(parsedCmd.butler.get(
"camera"))
408 parsedCmd)[0][
'expRefList'])
409 numCycles = int(numCcds/float(numCores) + 0.5)
410 return time*numExps*numCycles
413 def _makeArgumentParser(cls, *args, **kwargs):
414 kwargs.pop(
"doBatch",
False)
417 def run(self, expRefList, butler, calibId):
418 """!Construct a calib from a list of exposure references 420 This is the entry point, called by the TaskRunner.__call__ 422 Only the master node executes this method. 424 @param expRefList List of data references at the exposure level 425 @param butler Data butler 426 @param calibId Identifier dict for calib 428 for expRef
in expRefList:
429 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
433 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
436 outputIdItemList = list(outputId.items())
437 for ccdName
in ccdIdLists:
438 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
439 dataId.update(outputIdItemList)
441 dataId.update(outputIdItemList)
444 butler.get(self.
calibName +
"_filename", dataId)
445 except Exception
as e:
447 "Unable to determine output filename \"%s_filename\" from %s: %s" %
450 processPool =
Pool(
"process")
451 processPool.storeSet(butler=butler)
457 scales = self.
scale(ccdIdLists, data)
459 combinePool =
Pool(
"combine")
460 combinePool.storeSet(butler=butler)
463 calibs = self.
scatterCombine(combinePool, outputId, ccdIdLists, scales)
465 if self.config.doCameraImage:
466 camera = butler.get(
"camera")
470 butler.put(cameraImage, self.
calibName +
"_camera", dataId)
471 except Exception
as exc:
472 self.log.warn(
"Unable to create camera image: %s" % (exc,))
476 ccdIdLists = ccdIdLists,
479 processPool = processPool,
480 combinePool = combinePool,
484 """!Generate the data identifier for the output calib 486 The mean date and the common filter are included, using keywords 487 from the configuration. The CCD-specific part is not included 488 in the data identifier. 490 @param expRefList List of data references at exposure level 491 @param calibId Data identifier elements for the calib provided by the user 492 @return data identifier 496 for expRef
in expRefList:
497 butler = expRef.getButler()
498 dataId = expRef.dataId
500 midTime += self.
getMjd(butler, dataId)
503 if filterName
is None:
504 filterName = thisFilter
505 elif filterName != thisFilter:
506 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
507 dataId, thisFilter, filterName))
509 midTime /= len(expRefList)
510 date = str(dafBase.DateTime(
511 midTime, dafBase.DateTime.MJD).toPython().date())
513 outputId = {self.config.filter: filterName,
514 self.config.dateCalib: date}
515 outputId.update(calibId)
518 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
519 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier""" 520 if self.config.dateObs
in dataId:
521 dateObs = dataId[self.config.dateObs]
523 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
524 if "T" not in dateObs:
525 dateObs = dateObs +
"T12:00:00.0Z" 526 elif not dateObs.endswith(
"Z"):
529 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
532 """Determine the filter from a data identifier""" 533 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
537 if calibName
is None:
540 if missingKeys
is None:
541 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
543 for k
in missingKeys:
545 v = butler.queryMetadata(
'raw', [k], dataId)
546 except Exception
as e:
555 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
558 """!Update the metadata from the VisitInfo 560 \param calibImage The image whose metadata is to be set 561 \param exposureTime The exposure time for the image 562 \param darkTime The time since the last read (default: exposureTime) 566 darkTime = exposureTime
568 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
569 md = calibImage.getMetadata()
571 afwImage.setVisitInfoMetadata(md, visitInfo)
574 """!Scatter the processing among the nodes 576 We scatter each CCD independently (exposures aren't grouped together), 577 to make full use of all available processors. This necessitates piecing 578 everything back together in the same format as ccdIdLists afterwards. 580 Only the master node executes this method. 582 @param pool Process pool 583 @param ccdIdLists Dict of data identifier lists for each CCD name 584 @return Dict of lists of returned data for each CCD name 586 self.log.info(
"Scatter processing")
589 def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
590 """!Process a CCD, specified by a data identifier 592 After processing, optionally returns a result (produced by 593 the 'processResult' method) calculated from the processed 594 exposure. These results will be gathered by the master node, 595 and is a means for coordinated scaling of all CCDs for flats, 598 Only slave nodes execute this method. 600 @param cache Process pool cache 601 @param ccdId Data identifier for CCD 602 @param outputName Output dataset name for butler 603 @return result from 'processResult' 606 self.log.warn(
"Null identifier received on %s" % NODE)
609 if self.config.clobber
or not sensorRef.datasetExists(outputName):
610 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
613 except Exception
as e:
614 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
620 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
621 exposure = sensorRef.get(outputName)
625 """Process a single CCD, specified by a data reference 627 Generally, this simply means doing ISR. 629 Only slave nodes execute this method. 631 return self.isr.runDataRef(dataRef).exposure
634 """!Write the processed CCD 636 We need to write these out because we can't hold them all in 639 Only slave nodes execute this method. 641 @param dataRef Data reference 642 @param exposure CCD exposure to write 643 @param outputName Output dataset name for butler. 645 dataRef.put(exposure, outputName)
648 """Extract processing results from a processed exposure 650 This method generates what is gathered by the master node. 651 This can be a background measurement or similar for scaling 652 flat-fields. It must be picklable! 654 Only slave nodes execute this method. 659 """!Determine scaling across CCDs and exposures 661 This is necessary mainly for flats, so as to determine a 662 consistent scaling across the entire focal plane. This 663 implementation is simply a placeholder. 665 Only the master node executes this method. 667 @param ccdIdLists Dict of data identifier lists for each CCD tuple 668 @param data Dict of lists of returned data for each CCD tuple 669 @return dict of Struct(ccdScale: scaling for CCD, 670 expScales: scaling for each exposure 673 self.log.info(
"Scale on %s" % NODE)
674 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
675 for name
in ccdIdLists)
678 """!Scatter the combination of exposures across multiple nodes 680 In this case, we can only scatter across as many nodes as 683 Only the master node executes this method. 685 @param pool Process pool 686 @param outputId Output identifier (exposure part only) 687 @param ccdIdLists Dict of data identifier lists for each CCD name 688 @param scales Dict of structs with scales, for each CCD name 689 @param dict of binned images 691 self.log.info(
"Scatter combination")
692 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for 693 ccdName
in ccdIdLists]
694 images = pool.map(self.
combine, data, outputId)
695 return dict(zip(ccdIdLists.keys(), images))
698 """Get fully-qualified output data identifier 700 We may need to look up keys that aren't in the output dataId. 702 @param ccdName Name tuple for CCD 703 @param butler Data butler 704 @param outputId Data identifier for combined image (exposure part only) 705 @return fully-qualified output dataId 707 fullOutputId = {k: ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
708 fullOutputId.update(outputId)
710 fullOutputId.update(outputId)
714 """!Combine multiple exposures of a particular CCD and write the output 716 Only the slave nodes execute this method. 718 @param cache Process pool cache 719 @param struct Parameters for the combination, which has the following components: 720 * ccdName Name tuple for CCD 721 * ccdIdList List of data identifiers for combination 722 * scales Scales to apply (expScales are scalings for each exposure, 723 ccdScale is final scale for combined image) 724 @param outputId Data identifier for combined image (exposure part only) 725 @return binned calib image 728 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 729 dataId
in struct.ccdIdList]
730 self.log.info(
"Combining %s on %s" % (outputId, NODE))
731 calib = self.combination.
run(dataRefList, expScales=struct.scales.expScales,
732 finalScale=struct.scales.ccdScale)
734 if not hasattr(calib,
"getMetadata"):
735 if hasattr(calib,
"getVariance"):
736 calib = afwImage.makeExposure(calib)
738 calib = afwImage.DecoratedImageF(calib.getImage())
743 struct.ccdIdList, outputId)
747 self.
write(cache.butler, calib, outputId)
749 return afwMath.binImage(calib.getImage(), self.config.binning)
752 """!Record metadata including the inputs and creation details 754 This metadata will go into the FITS header. 756 @param butler Data butler 757 @param calib Combined calib exposure. 758 @param dataIdList List of data identifiers for calibration inputs 759 @param outputId Data identifier for output 761 header = calib.getMetadata()
765 now = time.localtime()
766 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
767 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
769 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
772 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if 774 for i, v
in enumerate(sorted(set(visits))):
775 header.add(
"CALIB_INPUT_%d" % (i,), v)
777 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
778 for key, value
in outputId.items()))
782 """Interpolate over NANs in the combined image 784 NANs can result from masked areas on the CCD. We don't want them getting 785 into our science images, so we replace them with the median of the image. 787 if hasattr(image,
"getMaskedImage"):
789 image = image.getMaskedImage().getImage()
790 if hasattr(image,
"getImage"):
791 image = image.getImage()
792 array = image.getArray()
793 bad = np.isnan(array)
794 array[bad] = np.median(array[np.logical_not(bad)])
796 def write(self, butler, exposure, dataId):
797 """!Write the final combined calib 799 Only the slave nodes execute this method 801 @param butler Data butler 802 @param exposure CCD exposure to write 803 @param dataId Data identifier for output 805 self.log.info(
"Writing %s on %s" % (dataId, NODE))
806 butler.put(exposure, self.
calibName, dataId)
809 """!Create and write an image of the entire camera 811 This is useful for judging the quality or getting an overview of 812 the features of the calib. 814 This requires that the 'ccd name' is a tuple containing only the 815 detector ID. If that is not the case, change CalibConfig.ccdKeys 816 or set CalibConfig.doCameraImage=False to disable this. 818 @param camera Camera object 819 @param dataId Data identifier for output 820 @param calibs Dict mapping 'ccd name' to calib image 823 class ImageSource(object):
824 """Source of images for makeImageFromCamera 826 This assumes that the 'ccd name' is a tuple containing 827 only the detector ID. 834 def getCcdImage(self, detector, imageFactory, binSize):
835 detId = (detector.getId(),)
836 if detId
not in self.
images:
837 return imageFactory(1, 1), detId
838 return self.
images[detId], detId
840 image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
841 binSize=self.config.binning)
845 """Configuration for bias construction. 847 No changes required compared to the base class, but 848 subclassed for distinction. 853 class BiasTask(CalibTask):
854 """Bias construction""" 855 ConfigClass = BiasConfig
856 _DefaultName =
"bias" 863 """Overrides to apply for bias construction""" 864 config.isr.doBias =
False 865 config.isr.doDark =
False 866 config.isr.doFlat =
False 867 config.isr.doFringe =
False 871 """Configuration for dark construction""" 872 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
873 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
874 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
875 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
876 repair = ConfigurableField(
877 target=RepairTask, doc=
"Task to repair artifacts")
880 CalibConfig.setDefaults(self)
887 The only major difference from the base class is a cosmic-ray 888 identification stage, and dividing each image by the dark time 889 to generate images of the dark rate. 891 ConfigClass = DarkConfig
892 _DefaultName =
"dark" 897 CalibTask.__init__(self, *args, **kwargs)
898 self.makeSubtask(
"repair")
902 """Overrides to apply for dark construction""" 903 config.isr.doDark =
False 904 config.isr.doFlat =
False 905 config.isr.doFringe =
False 908 """Process a single CCD 910 Besides the regular ISR, also masks cosmic-rays and divides each 911 processed image by the dark time to generate images of the dark rate. 912 The dark time is provided by the 'getDarkTime' method. 914 exposure = CalibTask.processSingle(self, sensorRef)
916 if self.config.doRepair:
917 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
918 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
920 self.repair.
run(exposure, keepCRs=
False)
921 if self.config.crGrow > 0:
922 mask = exposure.getMaskedImage().getMask().clone()
923 mask &= mask.getPlaneBitMask(
"CR")
924 fpSet = afwDet.FootprintSet(
925 mask, afwDet.Threshold(0.5))
926 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
927 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
929 mi = exposure.getMaskedImage()
934 """Retrieve the dark time for an exposure""" 935 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
936 if not np.isfinite(darkTime):
937 raise RuntimeError(
"Non-finite darkTime")
942 """Configuration for flat construction""" 943 iterations = Field(dtype=int, default=10,
944 doc=
"Number of iterations for scale determination")
945 stats = ConfigurableField(target=CalibStatsTask,
946 doc=
"Background statistics configuration")
952 The principal change from the base class involves gathering the background 953 values from each image and using them to determine the scalings for the final 956 ConfigClass = FlatConfig
957 _DefaultName =
"flat" 962 """Overrides for flat construction""" 963 config.isr.doFlat =
False 964 config.isr.doFringe =
False 967 CalibTask.__init__(self, *args, **kwargs)
968 self.makeSubtask(
"stats")
971 return self.stats.
run(exposure)
974 """Determine the scalings for the final combination 976 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling 977 of one CCD to all the others in an exposure, and E_j is the scaling 978 of the exposure. We convert everything to logarithms so we can work 979 with a linear system. We determine the C_i and E_j from B_ij by iteration, 980 under the additional constraint that the average CCD scale is unity. 982 This algorithm comes from Eugene Magnier and Pan-STARRS. 984 assert len(ccdIdLists.values()) > 0,
"No successful CCDs" 985 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
987 lengths) == 1,
"Number of successful exposures for each CCD differs" 988 assert tuple(lengths)[0] > 0,
"No successful exposures" 990 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
991 bgMatrix = np.array([[0.0] * len(expList)
992 for expList
in ccdIdLists.values()])
993 for name
in ccdIdLists:
996 d
if d
is not None else np.nan
for d
in data[name]]
998 numpyPrint = np.get_printoptions()
999 np.set_printoptions(threshold=np.inf)
1000 self.log.info(
"Input backgrounds: %s" % bgMatrix)
1003 numCcds = len(ccdIdLists)
1004 numExps = bgMatrix.shape[1]
1006 bgMatrix = np.log(bgMatrix)
1007 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1009 compScales = np.zeros(numCcds)
1010 expScales = np.array(
1011 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
1013 for iterate
in range(self.config.iterations):
1014 compScales = np.array(
1015 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
1016 expScales = np.array(
1017 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
1019 avgScale = np.average(np.exp(compScales))
1020 compScales -= np.log(avgScale)
1021 self.log.debug(
"Iteration %d exposure scales: %s",
1022 iterate, np.exp(expScales))
1023 self.log.debug(
"Iteration %d component scales: %s",
1024 iterate, np.exp(compScales))
1026 expScales = np.array(
1027 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
1029 if np.any(np.isnan(expScales)):
1030 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
1031 (bgMatrix, expScales))
1033 expScales = np.exp(expScales)
1034 compScales = np.exp(compScales)
1036 self.log.info(
"Exposure scales: %s" % expScales)
1037 self.log.info(
"Component relative scaling: %s" % compScales)
1038 np.set_printoptions(**numpyPrint)
1040 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1041 for ccdName
in ccdIdLists)
1045 """Configuration for fringe construction""" 1046 stats = ConfigurableField(target=CalibStatsTask,
1047 doc=
"Background statistics configuration")
1048 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1049 doc=
"Background configuration")
1050 detection = ConfigurableField(
1051 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1052 detectSigma = Field(dtype=float, default=1.0,
1053 doc=
"Detection PSF gaussian sigma")
1057 """Fringe construction task 1059 The principal change from the base class is that the images are 1060 background-subtracted and rescaled by the background. 1062 XXX This is probably not right for a straight-up combination, as we 1063 are currently doing, since the fringe amplitudes need not scale with 1066 XXX Would like to have this do PCA and generate multiple images, but 1067 that will take a bit of work with the persistence code. 1069 ConfigClass = FringeConfig
1070 _DefaultName =
"fringe" 1071 calibName =
"fringe" 1075 """Overrides for fringe construction""" 1076 config.isr.doFringe =
False 1079 CalibTask.__init__(self, *args, **kwargs)
1080 self.makeSubtask(
"detection")
1081 self.makeSubtask(
"stats")
1082 self.makeSubtask(
"subtractBackground")
1085 """Subtract the background and normalise by the background level""" 1086 exposure = CalibTask.processSingle(self, sensorRef)
1087 bgLevel = self.stats.
run(exposure)
1088 self.subtractBackground.
run(exposure)
1089 mi = exposure.getMaskedImage()
1091 footprintSets = self.detection.detectFootprints(
1092 exposure, sigma=self.config.detectSigma)
1093 mask = exposure.getMaskedImage().getMask()
1094 detected = 1 << mask.addMaskPlane(
"DETECTED")
1095 for fpSet
in (footprintSets.positive, footprintSets.negative):
1096 if fpSet
is not None:
1097 afwDet.setMaskFromFootprintList(
1098 mask, fpSet.getFootprints(), detected)
1103 """Configuration for sky frame construction""" 1104 detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1105 detectSigma = Field(dtype=float, default=2.0, doc=
"Detection PSF gaussian sigma")
1106 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1107 doc=
"Regular-scale background configuration, for object detection")
1108 largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1109 doc=
"Large-scale background configuration")
1110 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
1111 maskThresh = Field(dtype=float, default=3.0, doc=
"k-sigma threshold for masking pixels")
1112 mask = ListField(dtype=str, default=[
"BAD",
"SAT",
"DETECTED",
"NO_DATA"],
1113 doc=
"Mask planes to consider as contaminated")
1117 """Task for sky frame construction 1119 The sky frame is a (relatively) small-scale background 1120 model, the response of the camera to the sky. 1122 To construct, we first remove a large-scale background (e.g., caused 1123 by moonlight) which may vary from image to image. Then we construct a 1124 model of the sky, which is essentially a binned version of the image 1125 (important configuration parameters: sky.background.[xy]BinSize). 1126 It is these models which are coadded to yield the sky frame. 1128 ConfigClass = SkyConfig
1129 _DefaultName =
"sky" 1133 CalibTask.__init__(self, *args, **kwargs)
1134 self.makeSubtask(
"detection")
1135 self.makeSubtask(
"subtractBackground")
1136 self.makeSubtask(
"sky")
1139 """!Scatter the processing among the nodes 1141 Only the master node executes this method, assigning work to the 1144 We measure and subtract off a large-scale background model across 1145 all CCDs, which requires a scatter/gather. Then we process the 1146 individual CCDs, subtracting the large-scale background model and 1147 the residual background model measured. These residuals will be 1148 combined for the sky frame. 1150 @param pool Process pool 1151 @param ccdIdLists Dict of data identifier lists for each CCD name 1152 @return Dict of lists of returned data for each CCD name 1154 self.log.info(
"Scatter processing")
1156 numExps = set(len(expList)
for expList
in ccdIdLists.values())
1157 assert len(numExps) == 1
1158 numExps = numExps.pop()
1166 for exp
in range(numExps):
1167 bgModels = [bgModelList[ccdName][exp]
for ccdName
in ccdIdLists]
1168 visit = set(tuple(ccdIdLists[ccdName][exp][key]
for key
in sorted(self.config.visitKeys))
for 1169 ccdName
in ccdIdLists)
1170 assert len(visit) == 1
1172 bgModel = bgModels[0]
1173 for bg
in bgModels[1:]:
1175 self.log.info(
"Background model min/max for visit %s: %f %f", visit,
1176 np.min(bgModel.getStatsImage().getArray()),
1177 np.max(bgModel.getStatsImage().getArray()))
1178 backgrounds[visit] = bgModel
1179 scales[visit] = np.median(bgModel.getStatsImage().getArray())
1181 return mapToMatrix(pool, self.
process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1184 """!Measure background model for CCD 1186 This method is executed by the slaves. 1188 The background models for all CCDs in an exposure will be 1189 combined to form a full focal-plane background model. 1191 @param cache Process pool cache 1192 @param dataId Data identifier 1193 @return Bcakground model 1200 config = self.config.largeScaleBackground
1201 camera = dataRef.get(
"camera")
1202 bgModel = FocalPlaneBackground.fromCamera(config, camera)
1203 bgModel.addCcd(exposure)
1207 """!Process a single CCD for the background 1209 This method is executed by the slaves. 1211 Because we're interested in the background, we detect and mask astrophysical 1212 sources, and pixels above the noise level. 1214 @param dataRef Data reference for CCD. 1215 @return processed exposure 1217 if not self.config.clobber
and dataRef.datasetExists(
"postISRCCD"):
1218 return dataRef.get(
"postISRCCD")
1219 exposure = CalibTask.processSingle(self, dataRef)
1222 bgTemp = self.subtractBackground.
run(exposure).background
1223 footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1224 image = exposure.getMaskedImage()
1225 if footprints.background
is not None:
1226 image += footprints.background.getImageF()
1229 variance = image.getVariance()
1230 noise = np.sqrt(np.median(variance.getArray()))
1231 isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1232 image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask(
"DETECTED")
1235 image += bgTemp.getImage()
1238 maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1239 isBad = image.getMask().getArray() & maskVal > 0
1240 bgLevel = np.median(image.getImage().getArray()[~isBad])
1241 image.getImage().getArray()[isBad] = bgLevel
1242 dataRef.put(exposure,
"postISRCCD")
1246 """Process a single CCD, specified by a data reference 1248 We subtract the appropriate focal plane background model, 1249 divide by the appropriate scale and measure the background. 1251 Only slave nodes execute this method. 1253 @param dataRef Data reference for single CCD 1254 @param backgrounds Background model for each visit 1255 @param scales Scales for each visit 1256 @return Processed exposure 1258 visit = tuple(dataRef.dataId[key]
for key
in sorted(self.config.visitKeys))
1259 exposure = dataRef.get(
"postISRCCD", immediate=
True)
1260 image = exposure.getMaskedImage()
1261 detector = exposure.getDetector()
1262 bbox = image.getBBox()
1264 bgModel = backgrounds[visit]
1265 bg = bgModel.toCcdBackground(detector, bbox)
1266 image -= bg.getImage()
1267 image /= scales[visit]
1270 dataRef.put(bg,
"icExpBackground")
1274 """!Combine multiple background models of a particular CCD and write the output 1276 Only the slave nodes execute this method. 1278 @param cache Process pool cache 1279 @param struct Parameters for the combination, which has the following components: 1280 * ccdName Name tuple for CCD 1281 * ccdIdList List of data identifiers for combination 1282 @param outputId Data identifier for combined image (exposure part only) 1283 @return binned calib image 1286 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 1287 dataId
in struct.ccdIdList]
1288 self.log.info(
"Combining %s on %s" % (outputId, NODE))
1289 bgList = [dataRef.get(
"icExpBackground", immediate=
True).clone()
for dataRef
in dataRefList]
1291 bgExp = self.sky.averageBackgrounds(bgList)
1294 cache.butler.put(bgExp,
"sky", outputId)
1295 return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def applyOverrides(cls, config)
def getFilter(self, butler, dataId)
def __init__(self, args, kwargs)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def __init__(self, args, kwargs)
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def processResult(self, exposure)
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def __init__(self, calibName, args, kwargs)
def applyScale(self, exposure, scale=None)
def processSingle(self, sensorRef)
def applyOverrides(cls, config)
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def processSingle(self, sensorRef)
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def processSingle(self, dataRef)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def interpolateNans(self, image)
def getDataRef(butler, dataId, datasetType="raw")
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def scale(self, ccdIdLists, data)
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def parse_args(self, args, kwargs)
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def getDarkTime(self, exposure)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def __init__(self, args, kwargs)
def __init__(self, args, kwargs)
def applyOverrides(cls, config)
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def processResult(self, exposure)
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def __init__(self, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
def getTargetList(parsedCmd, kwargs)
def __init__(self, args, kwargs)
Base class for constructing calibs.
def applyOverrides(cls, config)
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)