1 from __future__
import absolute_import, division, print_function
10 from builtins
import zip
11 from builtins
import range
13 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
14 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
30 from .checksum
import checksum
31 from .utils
import getDataRef
35 """Parameters controlling the measurement of background statistics""" 36 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
37 default=int(afwMath.MEANCLIP))
38 clip = Field(doc=
"Clipping threshold for background",
39 dtype=float, default=3.0)
40 nIter = Field(doc=
"Clipping iterations for background",
42 mask = ListField(doc=
"Mask planes to reject",
43 dtype=str, default=[
"DETECTED",
"BAD",
"NO_DATA",])
47 """Measure statistics on the background 49 This can be useful for scaling the background, e.g., for flats and fringe frames. 51 ConfigClass = CalibStatsConfig
53 def run(self, exposureOrImage):
54 """!Measure a particular statistic on an image (of some sort). 56 @param exposureOrImage Exposure, MaskedImage or Image. 57 @return Value of desired statistic 59 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
60 afwImage.Mask.getPlaneBitMask(self.config.mask))
62 image = exposureOrImage.getMaskedImage()
65 image = exposureOrImage.getImage()
67 image = exposureOrImage
69 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
73 """Configuration for combining calib images""" 74 rows = Field(doc=
"Number of rows to read at a time",
75 dtype=int, default=512)
76 mask = ListField(doc=
"Mask planes to respect", dtype=str,
77 default=[
"SAT",
"DETECTED",
"INTRP"])
78 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
79 default=int(afwMath.MEANCLIP))
80 clip = Field(doc=
"Clipping threshold for combination",
81 dtype=float, default=3.0)
82 nIter = Field(doc=
"Clipping iterations for combination",
84 stats = ConfigurableField(target=CalibStatsTask,
85 doc=
"Background statistics configuration")
89 """Task to combine calib images""" 90 ConfigClass = CalibCombineConfig
93 Task.__init__(self, *args, **kwargs)
94 self.makeSubtask(
"stats")
96 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
97 """!Combine calib images for a single sensor 99 @param sensorRefList List of data references to combine (for a single sensor) 100 @param expScales List of scales to apply for each exposure 101 @param finalScale Desired scale for final combined image 102 @param inputName Data set name for inputs 103 @return combined image 106 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
107 afwImage.Mask.getPlaneBitMask(self.config.mask))
110 combined = afwImage.MaskedImageF(width, height)
111 numImages = len(sensorRefList)
112 imageList = [
None]*numImages
113 for start
in range(0, height, self.config.rows):
114 rows = min(self.config.rows, height - start)
115 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
116 afwGeom.Extent2I(width, rows))
117 subCombined = combined.Factory(combined, box)
119 for i, sensorRef
in enumerate(sensorRefList):
120 if sensorRef
is None:
123 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
124 if expScales
is not None:
126 imageList[i] = exposure.getMaskedImage()
128 self.
combine(subCombined, imageList, stats)
130 if finalScale
is not None:
131 background = self.stats.
run(combined)
132 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
133 (NODE, background, finalScale))
134 combined *= finalScale / background
139 """Get dimensions of the inputs""" 141 for sensorRef
in sensorRefList:
142 if sensorRef
is None:
144 md = sensorRef.get(inputName +
"_md")
145 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
149 """Apply scale to input exposure 151 This implementation applies a flux scaling: the input exposure is 152 divided by the provided scale. 154 if scale
is not None:
155 mi = exposure.getMaskedImage()
159 """!Combine multiple images 161 @param target Target image to receive the combined pixels 162 @param imageList List of input images 163 @param stats Statistics control 165 images = [img
for img
in imageList
if img
is not None]
166 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
170 """Determine a consistent size, given a list of image sizes""" 171 dim = set((w, h)
for w, h
in dimList)
174 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
179 """!Return a tuple of specific values from a dict 181 This provides a hashable representation of the dict from certain keywords. 182 This can be useful for creating e.g., a tuple of the values in the DataId 183 that identify the CCD. 185 @param dict_ dict to parse 186 @param keys keys to extract (order is important) 187 @return tuple of values 189 return tuple(dict_[k]
for k
in keys)
193 """!Determine a list of CCDs from exposure references 195 This essentially inverts the exposure-level references (which 196 provides a list of CCDs for each exposure), by providing 197 a dataId list for each CCD. Consider an input list of exposures 198 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this 201 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]} 203 This is a dict whose keys are tuples of the identifying values of a 204 CCD (usually just the CCD number) and the values are lists of dataIds 205 for that CCD in each exposure. A missing dataId is given the value 208 @param expRefList List of data references for exposures 209 @param level Level for the butler to generate CCDs 210 @param ccdKeys DataId keywords that identify a CCD 211 @return dict of data identifier lists for each CCD 213 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
214 level)]
for expRef
in expRefList]
217 ccdKeys = set(ccdKeys)
219 for ccdIdList
in expIdList:
220 for ccdId
in ccdIdList:
227 for n, ccdIdList
in enumerate(expIdList):
228 for ccdId
in ccdIdList:
230 if name
not in ccdLists:
232 ccdLists[name].append(ccdId)
236 ccdLists[ccd] = sorted(ccdLists[ccd], key=
lambda dd:
dictToTuple(dd, sorted(dd.keys())))
242 """Generate a matrix of results using pool.map 244 The function should have the call signature: 245 func(cache, dataId, *args, **kwargs) 247 We return a dict mapping 'ccd name' to a list of values for 250 @param pool Process pool 251 @param func Function to call for each dataId 252 @param ccdIdLists Dict of data identifier lists for each CCD name 253 @return matrix of results 255 dataIdList = sum(ccdIdLists.values(), [])
256 resultList = pool.map(func, dataIdList, *args, **kwargs)
258 data = dict((ccdName, [
None] * len(expList))
for ccdName, expList
in ccdIdLists.items())
259 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
260 for expNum, dataId
in enumerate(expList)]
261 for ccdName, expList
in ccdIdLists.items()], []))
262 for dataId, result
in zip(dataIdList, resultList):
265 ccdName, expNum = indices[tuple(dataId.values())]
266 data[ccdName][expNum] = result
271 """Split name=value pairs and put the result in a dict""" 273 def __call__(self, parser, namespace, values, option_string):
274 output = getattr(namespace, self.dest, {})
275 for nameValue
in values:
276 name, sep, valueStr = nameValue.partition(
"=")
278 parser.error(
"%s value %s must be in form name=value" %
279 (option_string, nameValue))
280 output[name] = valueStr
281 setattr(namespace, self.dest, output)
285 """ArgumentParser for calibration construction""" 288 """Add a --calibId argument to the standard pipe_base argument parser""" 289 ArgumentParser.__init__(self, *args, **kwargs)
291 self.add_id_argument(
"--id", datasetType=
"raw",
292 help=
"input identifiers, e.g., --id visit=123 ccd=4")
293 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
294 help=
"identifiers for calib, e.g., --calibId version=1",
295 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
300 Checks that the "--calibId" provided works. 302 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
304 keys = namespace.butler.getKeys(self.
calibName)
306 for name, value
in namespace.calibId.items():
309 "%s is not a relevant calib identifier key (%s)" % (name, keys))
310 parsed[name] = keys[name](value)
311 namespace.calibId = parsed
317 """Configuration for constructing calibs""" 318 clobber = Field(dtype=bool, default=
True,
319 doc=
"Clobber existing processed images?")
320 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
321 dateObs = Field(dtype=str, default=
"dateObs",
322 doc=
"Key for observation date in exposure registry")
323 dateCalib = Field(dtype=str, default=
"calibDate",
324 doc=
"Key for calib date in calib registry")
325 filter = Field(dtype=str, default=
"filter",
326 doc=
"Key for filter name in exposure/calib registries")
327 combination = ConfigurableField(
328 target=CalibCombineTask, doc=
"Calib combination configuration")
329 ccdKeys = ListField(dtype=str, default=[
"ccd"],
330 doc=
"DataId keywords specifying a CCD")
331 visitKeys = ListField(dtype=str, default=[
"visit"],
332 doc=
"DataId keywords specifying a visit")
333 calibKeys = ListField(dtype=str, default=[],
334 doc=
"DataId keywords specifying a calibration")
335 doCameraImage = Field(dtype=bool, default=
True, doc=
"Create camera overview image?")
336 binning = Field(dtype=int, default=64, doc=
"Binning to apply for camera image")
339 self.
isr.doWrite =
False 343 """Get parsed values into the CalibTask.run""" 346 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
349 """Call the Task with the kwargs from getTargetList""" 350 task = self.TaskClass(config=self.config, log=self.log)
353 result = task.run(**args)
356 result = task.run(**args)
357 except Exception
as e:
360 task.log.fatal(
"Failed: %s" % e)
361 traceback.print_exc(file=sys.stderr)
363 if self.doReturnResults:
365 exitStatus=exitStatus,
367 metadata=task.metadata,
372 exitStatus=exitStatus,
376 """!Base class for constructing calibs. 378 This should be subclassed for each of the required calib types. 379 The subclass should be sure to define the following class variables: 380 * _DefaultName: default name of the task, used by CmdLineTask 381 * calibName: name of the calibration data set in the butler 382 The subclass may optionally set: 383 * filterName: filter name to give the resultant calib 385 ConfigClass = CalibConfig
386 RunnerClass = CalibTaskRunner
393 BatchPoolTask.__init__(self, *args, **kwargs)
394 self.makeSubtask(
"isr")
395 self.makeSubtask(
"combination")
399 numCcds = len(parsedCmd.butler.get(
"camera"))
401 parsedCmd)[0][
'expRefList'])
402 numCycles = int(numCcds/float(numCores) + 0.5)
403 return time*numExps*numCycles
406 def _makeArgumentParser(cls, *args, **kwargs):
407 kwargs.pop(
"doBatch",
False)
410 def run(self, expRefList, butler, calibId):
411 """!Construct a calib from a list of exposure references 413 This is the entry point, called by the TaskRunner.__call__ 415 Only the master node executes this method. 417 @param expRefList List of data references at the exposure level 418 @param butler Data butler 419 @param calibId Identifier dict for calib 421 for expRef
in expRefList:
422 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
426 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
429 outputIdItemList = list(outputId.items())
430 for ccdName
in ccdIdLists:
431 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
432 dataId.update(outputIdItemList)
434 dataId.update(outputIdItemList)
437 butler.get(self.
calibName +
"_filename", dataId)
438 except Exception
as e:
440 "Unable to determine output filename \"%s_filename\" from %s: %s" %
443 processPool =
Pool(
"process")
444 processPool.storeSet(butler=butler)
450 scales = self.
scale(ccdIdLists, data)
452 combinePool =
Pool(
"combine")
453 combinePool.storeSet(butler=butler)
456 calibs = self.
scatterCombine(combinePool, outputId, ccdIdLists, scales)
458 if self.config.doCameraImage:
459 camera = butler.get(
"camera")
463 butler.put(cameraImage, self.
calibName +
"_camera", dataId)
464 except Exception
as exc:
465 self.log.warn(
"Unable to create camera image: %s" % (exc,))
469 ccdIdLists = ccdIdLists,
472 processPool = processPool,
473 combinePool = combinePool,
477 """!Generate the data identifier for the output calib 479 The mean date and the common filter are included, using keywords 480 from the configuration. The CCD-specific part is not included 481 in the data identifier. 483 @param expRefList List of data references at exposure level 484 @param calibId Data identifier elements for the calib provided by the user 485 @return data identifier 489 for expRef
in expRefList:
490 butler = expRef.getButler()
491 dataId = expRef.dataId
493 midTime += self.
getMjd(butler, dataId)
496 if filterName
is None:
497 filterName = thisFilter
498 elif filterName != thisFilter:
499 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
500 dataId, thisFilter, filterName))
502 midTime /= len(expRefList)
503 date = str(dafBase.DateTime(
504 midTime, dafBase.DateTime.MJD).toPython().date())
506 outputId = {self.config.filter: filterName,
507 self.config.dateCalib: date}
508 outputId.update(calibId)
511 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
512 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier""" 513 if self.config.dateObs
in dataId:
514 dateObs = dataId[self.config.dateObs]
516 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
517 if "T" not in dateObs:
518 dateObs = dateObs +
"T12:00:00.0Z" 519 elif not dateObs.endswith(
"Z"):
522 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
525 """Determine the filter from a data identifier""" 526 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
530 if calibName
is None:
533 if missingKeys
is None:
534 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
536 for k
in missingKeys:
538 v = butler.queryMetadata(
'raw', [k], dataId)
539 except Exception
as e:
548 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
551 """!Update the metadata from the VisitInfo 553 \param calibImage The image whose metadata is to be set 554 \param exposureTime The exposure time for the image 555 \param darkTime The time since the last read (default: exposureTime) 559 darkTime = exposureTime
561 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
562 md = calibImage.getMetadata()
564 afwImage.setVisitInfoMetadata(md, visitInfo)
567 """!Scatter the processing among the nodes 569 We scatter each CCD independently (exposures aren't grouped together), 570 to make full use of all available processors. This necessitates piecing 571 everything back together in the same format as ccdIdLists afterwards. 573 Only the master node executes this method. 575 @param pool Process pool 576 @param ccdIdLists Dict of data identifier lists for each CCD name 577 @return Dict of lists of returned data for each CCD name 579 self.log.info(
"Scatter processing")
582 def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
583 """!Process a CCD, specified by a data identifier 585 After processing, optionally returns a result (produced by 586 the 'processResult' method) calculated from the processed 587 exposure. These results will be gathered by the master node, 588 and is a means for coordinated scaling of all CCDs for flats, 591 Only slave nodes execute this method. 593 @param cache Process pool cache 594 @param ccdId Data identifier for CCD 595 @param outputName Output dataset name for butler 596 @return result from 'processResult' 599 self.log.warn(
"Null identifier received on %s" % NODE)
602 if self.config.clobber
or not sensorRef.datasetExists(outputName):
603 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
606 except Exception
as e:
607 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
613 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
614 exposure = sensorRef.get(outputName)
618 """Process a single CCD, specified by a data reference 620 Generally, this simply means doing ISR. 622 Only slave nodes execute this method. 624 return self.isr.runDataRef(dataRef).exposure
627 """!Write the processed CCD 629 We need to write these out because we can't hold them all in 632 Only slave nodes execute this method. 634 @param dataRef Data reference 635 @param exposure CCD exposure to write 636 @param outputName Output dataset name for butler. 638 dataRef.put(exposure, outputName)
641 """Extract processing results from a processed exposure 643 This method generates what is gathered by the master node. 644 This can be a background measurement or similar for scaling 645 flat-fields. It must be picklable! 647 Only slave nodes execute this method. 652 """!Determine scaling across CCDs and exposures 654 This is necessary mainly for flats, so as to determine a 655 consistent scaling across the entire focal plane. This 656 implementation is simply a placeholder. 658 Only the master node executes this method. 660 @param ccdIdLists Dict of data identifier lists for each CCD tuple 661 @param data Dict of lists of returned data for each CCD tuple 662 @return dict of Struct(ccdScale: scaling for CCD, 663 expScales: scaling for each exposure 666 self.log.info(
"Scale on %s" % NODE)
667 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
668 for name
in ccdIdLists)
671 """!Scatter the combination of exposures across multiple nodes 673 In this case, we can only scatter across as many nodes as 676 Only the master node executes this method. 678 @param pool Process pool 679 @param outputId Output identifier (exposure part only) 680 @param ccdIdLists Dict of data identifier lists for each CCD name 681 @param scales Dict of structs with scales, for each CCD name 682 @param dict of binned images 684 self.log.info(
"Scatter combination")
685 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for 686 ccdName
in ccdIdLists]
687 images = pool.map(self.
combine, data, outputId)
688 return dict(zip(ccdIdLists.keys(), images))
691 """Get fully-qualified output data identifier 693 We may need to look up keys that aren't in the output dataId. 695 @param ccdName Name tuple for CCD 696 @param butler Data butler 697 @param outputId Data identifier for combined image (exposure part only) 698 @return fully-qualified output dataId 700 fullOutputId = {k: ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
701 fullOutputId.update(outputId)
703 fullOutputId.update(outputId)
707 """!Combine multiple exposures of a particular CCD and write the output 709 Only the slave nodes execute this method. 711 @param cache Process pool cache 712 @param struct Parameters for the combination, which has the following components: 713 * ccdName Name tuple for CCD 714 * ccdIdList List of data identifiers for combination 715 * scales Scales to apply (expScales are scalings for each exposure, 716 ccdScale is final scale for combined image) 717 @param outputId Data identifier for combined image (exposure part only) 718 @return binned calib image 721 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 722 dataId
in struct.ccdIdList]
723 self.log.info(
"Combining %s on %s" % (outputId, NODE))
724 calib = self.combination.
run(dataRefList, expScales=struct.scales.expScales,
725 finalScale=struct.scales.ccdScale)
727 if not hasattr(calib,
"getMetadata"):
728 if hasattr(calib,
"getVariance"):
729 calib = afwImage.makeExposure(calib)
731 calib = afwImage.DecoratedImageF(calib.getImage())
736 struct.ccdIdList, outputId)
740 self.
write(cache.butler, calib, outputId)
742 return afwMath.binImage(calib.getImage(), self.config.binning)
745 """!Record metadata including the inputs and creation details 747 This metadata will go into the FITS header. 749 @param butler Data butler 750 @param calib Combined calib exposure. 751 @param dataIdList List of data identifiers for calibration inputs 752 @param outputId Data identifier for output 754 header = calib.getMetadata()
758 now = time.localtime()
759 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
760 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
762 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
765 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if 767 for i, v
in enumerate(sorted(set(visits))):
768 header.add(
"CALIB_INPUT_%d" % (i,), v)
770 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
771 for key, value
in outputId.items()))
775 """Interpolate over NANs in the combined image 777 NANs can result from masked areas on the CCD. We don't want them getting 778 into our science images, so we replace them with the median of the image. 780 if hasattr(image,
"getMaskedImage"):
782 image = image.getMaskedImage().getImage()
783 if hasattr(image,
"getImage"):
784 image = image.getImage()
785 array = image.getArray()
786 bad = np.isnan(array)
787 array[bad] = np.median(array[np.logical_not(bad)])
789 def write(self, butler, exposure, dataId):
790 """!Write the final combined calib 792 Only the slave nodes execute this method 794 @param butler Data butler 795 @param exposure CCD exposure to write 796 @param dataId Data identifier for output 798 self.log.info(
"Writing %s on %s" % (dataId, NODE))
799 butler.put(exposure, self.
calibName, dataId)
802 """!Create and write an image of the entire camera 804 This is useful for judging the quality or getting an overview of 805 the features of the calib. 807 This requires that the 'ccd name' is a tuple containing only the 808 detector ID. If that is not the case, change CalibConfig.ccdKeys 809 or set CalibConfig.doCameraImage=False to disable this. 811 @param camera Camera object 812 @param dataId Data identifier for output 813 @param calibs Dict mapping 'ccd name' to calib image 816 class ImageSource(object):
817 """Source of images for makeImageFromCamera 819 This assumes that the 'ccd name' is a tuple containing 820 only the detector ID. 827 def getCcdImage(self, detector, imageFactory, binSize):
828 detId = (detector.getId(),)
829 if detId
not in self.
images:
830 return imageFactory(1, 1), detId
831 return self.
images[detId], detId
833 image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
834 binSize=self.config.binning)
838 """Configuration for bias construction. 840 No changes required compared to the base class, but 841 subclassed for distinction. 846 class BiasTask(CalibTask):
847 """Bias construction""" 848 ConfigClass = BiasConfig
849 _DefaultName =
"bias" 856 """Overrides to apply for bias construction""" 857 config.isr.doBias =
False 858 config.isr.doDark =
False 859 config.isr.doFlat =
False 860 config.isr.doFringe =
False 864 """Configuration for dark construction""" 865 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
866 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
867 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
868 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
869 repair = ConfigurableField(
870 target=RepairTask, doc=
"Task to repair artifacts")
873 CalibConfig.setDefaults(self)
880 The only major difference from the base class is a cosmic-ray 881 identification stage, and dividing each image by the dark time 882 to generate images of the dark rate. 884 ConfigClass = DarkConfig
885 _DefaultName =
"dark" 890 CalibTask.__init__(self, *args, **kwargs)
891 self.makeSubtask(
"repair")
895 """Overrides to apply for dark construction""" 896 config.isr.doDark =
False 897 config.isr.doFlat =
False 898 config.isr.doFringe =
False 901 """Process a single CCD 903 Besides the regular ISR, also masks cosmic-rays and divides each 904 processed image by the dark time to generate images of the dark rate. 905 The dark time is provided by the 'getDarkTime' method. 907 exposure = CalibTask.processSingle(self, sensorRef)
909 if self.config.doRepair:
910 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
911 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
913 self.repair.
run(exposure, keepCRs=
False)
914 if self.config.crGrow > 0:
915 mask = exposure.getMaskedImage().getMask().clone()
916 mask &= mask.getPlaneBitMask(
"CR")
917 fpSet = afwDet.FootprintSet(
918 mask, afwDet.Threshold(0.5))
919 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
920 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
922 mi = exposure.getMaskedImage()
927 """Retrieve the dark time for an exposure""" 928 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
929 if not np.isfinite(darkTime):
930 raise RuntimeError(
"Non-finite darkTime")
935 """Configuration for flat construction""" 936 iterations = Field(dtype=int, default=10,
937 doc=
"Number of iterations for scale determination")
938 stats = ConfigurableField(target=CalibStatsTask,
939 doc=
"Background statistics configuration")
945 The principal change from the base class involves gathering the background 946 values from each image and using them to determine the scalings for the final 949 ConfigClass = FlatConfig
950 _DefaultName =
"flat" 955 """Overrides for flat construction""" 956 config.isr.doFlat =
False 957 config.isr.doFringe =
False 960 CalibTask.__init__(self, *args, **kwargs)
961 self.makeSubtask(
"stats")
964 return self.stats.
run(exposure)
967 """Determine the scalings for the final combination 969 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling 970 of one CCD to all the others in an exposure, and E_j is the scaling 971 of the exposure. We convert everything to logarithms so we can work 972 with a linear system. We determine the C_i and E_j from B_ij by iteration, 973 under the additional constraint that the average CCD scale is unity. 975 This algorithm comes from Eugene Magnier and Pan-STARRS. 977 assert len(ccdIdLists.values()) > 0,
"No successful CCDs" 978 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
980 lengths) == 1,
"Number of successful exposures for each CCD differs" 981 assert tuple(lengths)[0] > 0,
"No successful exposures" 983 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
984 bgMatrix = np.array([[0.0] * len(expList)
985 for expList
in ccdIdLists.values()])
986 for name
in ccdIdLists:
989 d
if d
is not None else np.nan
for d
in data[name]]
991 numpyPrint = np.get_printoptions()
992 np.set_printoptions(threshold=np.inf)
993 self.log.info(
"Input backgrounds: %s" % bgMatrix)
996 numCcds = len(ccdIdLists)
997 numExps = bgMatrix.shape[1]
999 bgMatrix = np.log(bgMatrix)
1000 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1002 compScales = np.zeros(numCcds)
1003 expScales = np.array(
1004 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
1006 for iterate
in range(self.config.iterations):
1007 compScales = np.array(
1008 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
1009 expScales = np.array(
1010 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
1012 avgScale = np.average(np.exp(compScales))
1013 compScales -= np.log(avgScale)
1014 self.log.debug(
"Iteration %d exposure scales: %s",
1015 iterate, np.exp(expScales))
1016 self.log.debug(
"Iteration %d component scales: %s",
1017 iterate, np.exp(compScales))
1019 expScales = np.array(
1020 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
1022 if np.any(np.isnan(expScales)):
1023 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
1024 (bgMatrix, expScales))
1026 expScales = np.exp(expScales)
1027 compScales = np.exp(compScales)
1029 self.log.info(
"Exposure scales: %s" % expScales)
1030 self.log.info(
"Component relative scaling: %s" % compScales)
1031 np.set_printoptions(**numpyPrint)
1033 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1034 for ccdName
in ccdIdLists)
1038 """Configuration for fringe construction""" 1039 stats = ConfigurableField(target=CalibStatsTask,
1040 doc=
"Background statistics configuration")
1041 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1042 doc=
"Background configuration")
1043 detection = ConfigurableField(
1044 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1045 detectSigma = Field(dtype=float, default=1.0,
1046 doc=
"Detection PSF gaussian sigma")
1050 """Fringe construction task 1052 The principal change from the base class is that the images are 1053 background-subtracted and rescaled by the background. 1055 XXX This is probably not right for a straight-up combination, as we 1056 are currently doing, since the fringe amplitudes need not scale with 1059 XXX Would like to have this do PCA and generate multiple images, but 1060 that will take a bit of work with the persistence code. 1062 ConfigClass = FringeConfig
1063 _DefaultName =
"fringe" 1064 calibName =
"fringe" 1068 """Overrides for fringe construction""" 1069 config.isr.doFringe =
False 1072 CalibTask.__init__(self, *args, **kwargs)
1073 self.makeSubtask(
"detection")
1074 self.makeSubtask(
"stats")
1075 self.makeSubtask(
"subtractBackground")
1078 """Subtract the background and normalise by the background level""" 1079 exposure = CalibTask.processSingle(self, sensorRef)
1080 bgLevel = self.stats.
run(exposure)
1081 self.subtractBackground.
run(exposure)
1082 mi = exposure.getMaskedImage()
1084 footprintSets = self.detection.detectFootprints(
1085 exposure, sigma=self.config.detectSigma)
1086 mask = exposure.getMaskedImage().getMask()
1087 detected = 1 << mask.addMaskPlane(
"DETECTED")
1088 for fpSet
in (footprintSets.positive, footprintSets.negative):
1089 if fpSet
is not None:
1090 afwDet.setMaskFromFootprintList(
1091 mask, fpSet.getFootprints(), detected)
1096 """Configuration for sky frame construction""" 1097 detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1098 detectSigma = Field(dtype=float, default=2.0, doc=
"Detection PSF gaussian sigma")
1099 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1100 doc=
"Regular-scale background configuration, for object detection")
1101 largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1102 doc=
"Large-scale background configuration")
1103 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
1104 maskThresh = Field(dtype=float, default=3.0, doc=
"k-sigma threshold for masking pixels")
1105 mask = ListField(dtype=str, default=[
"BAD",
"SAT",
"DETECTED",
"NO_DATA"],
1106 doc=
"Mask planes to consider as contaminated")
1110 """Task for sky frame construction 1112 The sky frame is a (relatively) small-scale background 1113 model, the response of the camera to the sky. 1115 To construct, we first remove a large-scale background (e.g., caused 1116 by moonlight) which may vary from image to image. Then we construct a 1117 model of the sky, which is essentially a binned version of the image 1118 (important configuration parameters: sky.background.[xy]BinSize). 1119 It is these models which are coadded to yield the sky frame. 1121 ConfigClass = SkyConfig
1122 _DefaultName =
"sky" 1126 CalibTask.__init__(self, *args, **kwargs)
1127 self.makeSubtask(
"detection")
1128 self.makeSubtask(
"subtractBackground")
1129 self.makeSubtask(
"sky")
1132 """!Scatter the processing among the nodes 1134 Only the master node executes this method, assigning work to the 1137 We measure and subtract off a large-scale background model across 1138 all CCDs, which requires a scatter/gather. Then we process the 1139 individual CCDs, subtracting the large-scale background model and 1140 the residual background model measured. These residuals will be 1141 combined for the sky frame. 1143 @param pool Process pool 1144 @param ccdIdLists Dict of data identifier lists for each CCD name 1145 @return Dict of lists of returned data for each CCD name 1147 self.log.info(
"Scatter processing")
1149 numExps = set(len(expList)
for expList
in ccdIdLists.values())
1150 assert len(numExps) == 1
1151 numExps = numExps.pop()
1159 for exp
in range(numExps):
1160 bgModels = [bgModelList[ccdName][exp]
for ccdName
in ccdIdLists]
1161 visit = set(tuple(ccdIdLists[ccdName][exp][key]
for key
in sorted(self.config.visitKeys))
for 1162 ccdName
in ccdIdLists)
1163 assert len(visit) == 1
1165 bgModel = bgModels[0]
1166 for bg
in bgModels[1:]:
1168 self.log.info(
"Background model min/max for visit %s: %f %f", visit,
1169 np.min(bgModel.getStatsImage().getArray()),
1170 np.max(bgModel.getStatsImage().getArray()))
1171 backgrounds[visit] = bgModel
1172 scales[visit] = np.median(bgModel.getStatsImage().getArray())
1174 return mapToMatrix(pool, self.
process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1177 """!Measure background model for CCD 1179 This method is executed by the slaves. 1181 The background models for all CCDs in an exposure will be 1182 combined to form a full focal-plane background model. 1184 @param cache Process pool cache 1185 @param dataId Data identifier 1186 @return Bcakground model 1193 config = self.config.largeScaleBackground
1194 camera = dataRef.get(
"camera")
1195 bgModel = FocalPlaneBackground.fromCamera(config, camera)
1196 bgModel.addCcd(exposure)
1200 """!Process a single CCD for the background 1202 This method is executed by the slaves. 1204 Because we're interested in the background, we detect and mask astrophysical 1205 sources, and pixels above the noise level. 1207 @param dataRef Data reference for CCD. 1208 @return processed exposure 1210 if not self.config.clobber
and dataRef.datasetExists(
"postISRCCD"):
1211 return dataRef.get(
"postISRCCD")
1212 exposure = CalibTask.processSingle(self, dataRef)
1215 bgTemp = self.subtractBackground.
run(exposure).background
1216 footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1217 image = exposure.getMaskedImage()
1218 if footprints.background
is not None:
1219 image += footprints.background.getImageF()
1222 variance = image.getVariance()
1223 noise = np.sqrt(np.median(variance.getArray()))
1224 isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1225 image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask(
"DETECTED")
1228 image += bgTemp.getImage()
1231 maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1232 isBad = image.getMask().getArray() & maskVal > 0
1233 bgLevel = np.median(image.getImage().getArray()[~isBad])
1234 image.getImage().getArray()[isBad] = bgLevel
1235 dataRef.put(exposure,
"postISRCCD")
1239 """Process a single CCD, specified by a data reference 1241 We subtract the appropriate focal plane background model, 1242 divide by the appropriate scale and measure the background. 1244 Only slave nodes execute this method. 1246 @param dataRef Data reference for single CCD 1247 @param backgrounds Background model for each visit 1248 @param scales Scales for each visit 1249 @return Processed exposure 1251 visit = tuple(dataRef.dataId[key]
for key
in sorted(self.config.visitKeys))
1252 exposure = dataRef.get(
"postISRCCD", immediate=
True)
1253 image = exposure.getMaskedImage()
1254 detector = exposure.getDetector()
1255 bbox = image.getBBox()
1257 bgModel = backgrounds[visit]
1258 bg = bgModel.toCcdBackground(detector, bbox)
1259 image -= bg.getImage()
1260 image /= scales[visit]
1263 dataRef.put(bg,
"icExpBackground")
1267 """!Combine multiple background models of a particular CCD and write the output 1269 Only the slave nodes execute this method. 1271 @param cache Process pool cache 1272 @param struct Parameters for the combination, which has the following components: 1273 * ccdName Name tuple for CCD 1274 * ccdIdList List of data identifiers for combination 1275 @param outputId Data identifier for combined image (exposure part only) 1276 @return binned calib image 1279 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 1280 dataId
in struct.ccdIdList]
1281 self.log.info(
"Combining %s on %s" % (outputId, NODE))
1282 bgList = [dataRef.get(
"icExpBackground", immediate=
True).clone()
for dataRef
in dataRefList]
1284 bgExp = self.sky.averageBackgrounds(bgList)
1287 cache.butler.put(bgExp,
"sky", outputId)
1288 return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def applyOverrides(cls, config)
def getFilter(self, butler, dataId)
def __init__(self, args, kwargs)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def __init__(self, args, kwargs)
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def processResult(self, exposure)
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def __init__(self, calibName, args, kwargs)
def applyScale(self, exposure, scale=None)
def processSingle(self, sensorRef)
def applyOverrides(cls, config)
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def processSingle(self, sensorRef)
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def processSingle(self, dataRef)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def interpolateNans(self, image)
def getDataRef(butler, dataId, datasetType="raw")
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def scale(self, ccdIdLists, data)
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def parse_args(self, args, kwargs)
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def getDarkTime(self, exposure)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def __init__(self, args, kwargs)
def __init__(self, args, kwargs)
def applyOverrides(cls, config)
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def processResult(self, exposure)
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def __init__(self, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
def getTargetList(parsedCmd, kwargs)
def __init__(self, args, kwargs)
Base class for constructing calibs.
def applyOverrides(cls, config)
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)