1 from __future__
import absolute_import, division, print_function
11 from builtins
import zip
12 from builtins
import range
14 from lsst.pex.config import Config, ConfigurableField, Field, ListField, ConfigField
15 from lsst.pipe.base import Task, Struct, TaskRunner, ArgumentParser
31 from .checksum
import checksum
32 from .utils
import getDataRef
36 """Parameters controlling the measurement of background statistics""" 37 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
38 default=int(afwMath.MEANCLIP))
39 clip = Field(doc=
"Clipping threshold for background",
40 dtype=float, default=3.0)
41 nIter = Field(doc=
"Clipping iterations for background",
43 maxVisitsToCalcErrorFromInputVariance = Field(
44 doc=
"Maximum number of visits to estimate variance from input variance, not per-pixel spread",
46 mask = ListField(doc=
"Mask planes to reject",
47 dtype=str, default=[
"DETECTED",
"BAD",
"NO_DATA",])
51 """Measure statistics on the background 53 This can be useful for scaling the background, e.g., for flats and fringe frames. 55 ConfigClass = CalibStatsConfig
57 def run(self, exposureOrImage):
58 """!Measure a particular statistic on an image (of some sort). 60 @param exposureOrImage Exposure, MaskedImage or Image. 61 @return Value of desired statistic 63 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
64 afwImage.Mask.getPlaneBitMask(self.config.mask))
66 image = exposureOrImage.getMaskedImage()
69 image = exposureOrImage.getImage()
71 image = exposureOrImage
73 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
77 """Configuration for combining calib images""" 78 rows = Field(doc=
"Number of rows to read at a time",
79 dtype=int, default=512)
80 mask = ListField(doc=
"Mask planes to respect", dtype=str,
81 default=[
"SAT",
"DETECTED",
"INTRP"])
82 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
83 default=int(afwMath.MEANCLIP))
84 clip = Field(doc=
"Clipping threshold for combination",
85 dtype=float, default=3.0)
86 nIter = Field(doc=
"Clipping iterations for combination",
88 stats = ConfigurableField(target=CalibStatsTask,
89 doc=
"Background statistics configuration")
93 """Task to combine calib images""" 94 ConfigClass = CalibCombineConfig
97 Task.__init__(self, *args, **kwargs)
98 self.makeSubtask(
"stats")
100 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
101 """!Combine calib images for a single sensor 103 @param sensorRefList List of data references to combine (for a single sensor) 104 @param expScales List of scales to apply for each exposure 105 @param finalScale Desired scale for final combined image 106 @param inputName Data set name for inputs 107 @return combined image 110 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
111 afwImage.Mask.getPlaneBitMask(self.config.mask))
112 numImages = len(sensorRefList)
113 if numImages < self.config.stats.maxVisitsToCalcErrorFromInputVariance:
114 stats.setCalcErrorFromInputVariance(
True)
117 combined = afwImage.MaskedImageF(width, height)
118 imageList = [
None]*numImages
119 for start
in range(0, height, self.config.rows):
120 rows = min(self.config.rows, height - start)
121 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
122 afwGeom.Extent2I(width, rows))
123 subCombined = combined.Factory(combined, box)
125 for i, sensorRef
in enumerate(sensorRefList):
126 if sensorRef
is None:
129 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
130 if expScales
is not None:
132 imageList[i] = exposure.getMaskedImage()
134 self.
combine(subCombined, imageList, stats)
136 if finalScale
is not None:
137 background = self.stats.
run(combined)
138 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
139 (NODE, background, finalScale))
140 combined *= finalScale / background
145 """Get dimensions of the inputs""" 147 for sensorRef
in sensorRefList:
148 if sensorRef
is None:
150 md = sensorRef.get(inputName +
"_md")
151 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
155 """Apply scale to input exposure 157 This implementation applies a flux scaling: the input exposure is 158 divided by the provided scale. 160 if scale
is not None:
161 mi = exposure.getMaskedImage()
165 """!Combine multiple images 167 @param target Target image to receive the combined pixels 168 @param imageList List of input images 169 @param stats Statistics control 171 images = [img
for img
in imageList
if img
is not None]
172 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
176 """Determine a consistent size, given a list of image sizes""" 177 dim = set((w, h)
for w, h
in dimList)
180 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
185 """!Return a tuple of specific values from a dict 187 This provides a hashable representation of the dict from certain keywords. 188 This can be useful for creating e.g., a tuple of the values in the DataId 189 that identify the CCD. 191 @param dict_ dict to parse 192 @param keys keys to extract (order is important) 193 @return tuple of values 195 return tuple(dict_[k]
for k
in keys)
199 """!Determine a list of CCDs from exposure references 201 This essentially inverts the exposure-level references (which 202 provides a list of CCDs for each exposure), by providing 203 a dataId list for each CCD. Consider an input list of exposures 204 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this 207 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]} 209 This is a dict whose keys are tuples of the identifying values of a 210 CCD (usually just the CCD number) and the values are lists of dataIds 211 for that CCD in each exposure. A missing dataId is given the value 214 @param expRefList List of data references for exposures 215 @param level Level for the butler to generate CCDs 216 @param ccdKeys DataId keywords that identify a CCD 217 @return dict of data identifier lists for each CCD; 218 keys are values of ccdKeys in order 220 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
221 level)]
for expRef
in expRefList]
224 if len(ccdKeys) != len(set(ccdKeys)):
225 raise RuntimeError(
"Duplicate keys found in ccdKeys: %s" % ccdKeys)
227 for ccdIdList
in expIdList:
228 for ccdId
in ccdIdList:
235 for n, ccdIdList
in enumerate(expIdList):
236 for ccdId
in ccdIdList:
238 if name
not in ccdLists:
240 ccdLists[name].append(ccdId)
244 ccdLists[ccd] = sorted(ccdLists[ccd], key=
lambda dd:
dictToTuple(dd, sorted(dd.keys())))
250 """Generate a matrix of results using pool.map 252 The function should have the call signature: 253 func(cache, dataId, *args, **kwargs) 255 We return a dict mapping 'ccd name' to a list of values for 258 @param pool Process pool 259 @param func Function to call for each dataId 260 @param ccdIdLists Dict of data identifier lists for each CCD name 261 @return matrix of results 263 dataIdList = sum(ccdIdLists.values(), [])
264 resultList = pool.map(func, dataIdList, *args, **kwargs)
266 data = dict((ccdName, [
None] * len(expList))
for ccdName, expList
in ccdIdLists.items())
267 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
268 for expNum, dataId
in enumerate(expList)]
269 for ccdName, expList
in ccdIdLists.items()], []))
270 for dataId, result
in zip(dataIdList, resultList):
273 ccdName, expNum = indices[tuple(dataId.values())]
274 data[ccdName][expNum] = result
279 """Split name=value pairs and put the result in a dict""" 281 def __call__(self, parser, namespace, values, option_string):
282 output = getattr(namespace, self.dest, {})
283 for nameValue
in values:
284 name, sep, valueStr = nameValue.partition(
"=")
286 parser.error(
"%s value %s must be in form name=value" %
287 (option_string, nameValue))
288 output[name] = valueStr
289 setattr(namespace, self.dest, output)
293 """ArgumentParser for calibration construction""" 296 """Add a --calibId argument to the standard pipe_base argument parser""" 297 ArgumentParser.__init__(self, *args, **kwargs)
299 self.add_id_argument(
"--id", datasetType=
"raw",
300 help=
"input identifiers, e.g., --id visit=123 ccd=4")
301 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
302 help=
"identifiers for calib, e.g., --calibId version=1",
303 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
308 Checks that the "--calibId" provided works. 310 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
312 keys = namespace.butler.getKeys(self.
calibName)
314 for name, value
in namespace.calibId.items():
317 "%s is not a relevant calib identifier key (%s)" % (name, keys))
318 parsed[name] = keys[name](value)
319 namespace.calibId = parsed
325 """Configuration for constructing calibs""" 326 clobber = Field(dtype=bool, default=
True,
327 doc=
"Clobber existing processed images?")
328 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
329 dateObs = Field(dtype=str, default=
"dateObs",
330 doc=
"Key for observation date in exposure registry")
331 dateCalib = Field(dtype=str, default=
"calibDate",
332 doc=
"Key for calib date in calib registry")
333 filter = Field(dtype=str, default=
"filter",
334 doc=
"Key for filter name in exposure/calib registries")
335 combination = ConfigurableField(
336 target=CalibCombineTask, doc=
"Calib combination configuration")
337 ccdKeys = ListField(dtype=str, default=[
"ccd"],
338 doc=
"DataId keywords specifying a CCD")
339 visitKeys = ListField(dtype=str, default=[
"visit"],
340 doc=
"DataId keywords specifying a visit")
341 calibKeys = ListField(dtype=str, default=[],
342 doc=
"DataId keywords specifying a calibration")
343 doCameraImage = Field(dtype=bool, default=
True, doc=
"Create camera overview image?")
344 binning = Field(dtype=int, default=64, doc=
"Binning to apply for camera image")
347 self.
isr.doWrite =
False 351 """Get parsed values into the CalibTask.run""" 354 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
357 """Call the Task with the kwargs from getTargetList""" 358 task = self.TaskClass(config=self.config, log=self.log)
361 result = task.run(**args)
364 result = task.run(**args)
365 except Exception
as e:
368 task.log.fatal(
"Failed: %s" % e)
369 traceback.print_exc(file=sys.stderr)
371 if self.doReturnResults:
373 exitStatus=exitStatus,
375 metadata=task.metadata,
380 exitStatus=exitStatus,
384 """!Base class for constructing calibs. 386 This should be subclassed for each of the required calib types. 387 The subclass should be sure to define the following class variables: 388 * _DefaultName: default name of the task, used by CmdLineTask 389 * calibName: name of the calibration data set in the butler 390 The subclass may optionally set: 391 * filterName: filter name to give the resultant calib 393 ConfigClass = CalibConfig
394 RunnerClass = CalibTaskRunner
401 BatchPoolTask.__init__(self, *args, **kwargs)
402 self.makeSubtask(
"isr")
403 self.makeSubtask(
"combination")
407 numCcds = len(parsedCmd.butler.get(
"camera"))
409 parsedCmd)[0][
'expRefList'])
410 numCycles = int(numCcds/float(numCores) + 0.5)
411 return time*numExps*numCycles
414 def _makeArgumentParser(cls, *args, **kwargs):
415 kwargs.pop(
"doBatch",
False)
418 def run(self, expRefList, butler, calibId):
419 """!Construct a calib from a list of exposure references 421 This is the entry point, called by the TaskRunner.__call__ 423 Only the master node executes this method. 425 @param expRefList List of data references at the exposure level 426 @param butler Data butler 427 @param calibId Identifier dict for calib 429 for expRef
in expRefList:
430 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
434 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
438 outputIdItemList = list(outputId.items())
439 for ccdName
in ccdIdLists:
440 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
441 dataId.update(outputIdItemList)
443 dataId.update(outputIdItemList)
446 butler.get(self.
calibName +
"_filename", dataId)
447 except Exception
as e:
449 "Unable to determine output filename \"%s_filename\" from %s: %s" %
452 processPool =
Pool(
"process")
453 processPool.storeSet(butler=butler)
459 scales = self.
scale(ccdIdLists, data)
461 combinePool =
Pool(
"combine")
462 combinePool.storeSet(butler=butler)
465 calibs = self.
scatterCombine(combinePool, outputId, ccdIdLists, scales)
467 if self.config.doCameraImage:
468 camera = butler.get(
"camera")
470 calibs = {butler.get(
"postISRCCD_detector",
471 dict(zip(self.config.ccdKeys, ccdName))).getId(): calibs[ccdName]
472 for ccdName
in ccdIdLists}
476 butler.put(cameraImage, self.
calibName +
"_camera", dataId)
477 except Exception
as exc:
478 self.log.warn(
"Unable to create camera image: %s" % (exc,))
482 ccdIdLists = ccdIdLists,
485 processPool = processPool,
486 combinePool = combinePool,
490 """!Generate the data identifier for the output calib 492 The mean date and the common filter are included, using keywords 493 from the configuration. The CCD-specific part is not included 494 in the data identifier. 496 @param expRefList List of data references at exposure level 497 @param calibId Data identifier elements for the calib provided by the user 498 @return data identifier 502 for expRef
in expRefList:
503 butler = expRef.getButler()
504 dataId = expRef.dataId
506 midTime += self.
getMjd(butler, dataId)
509 if filterName
is None:
510 filterName = thisFilter
511 elif filterName != thisFilter:
512 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
513 dataId, thisFilter, filterName))
515 midTime /= len(expRefList)
516 date = str(dafBase.DateTime(
517 midTime, dafBase.DateTime.MJD).toPython().date())
519 outputId = {self.config.filter: filterName,
520 self.config.dateCalib: date}
521 outputId.update(calibId)
524 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
525 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier""" 526 if self.config.dateObs
in dataId:
527 dateObs = dataId[self.config.dateObs]
529 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
530 if "T" not in dateObs:
531 dateObs = dateObs +
"T12:00:00.0Z" 532 elif not dateObs.endswith(
"Z"):
535 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
538 """Determine the filter from a data identifier""" 539 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
543 if calibName
is None:
546 if missingKeys
is None:
547 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
549 for k
in missingKeys:
551 v = butler.queryMetadata(
'raw', [k], dataId)
552 except Exception
as e:
561 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
564 """!Update the metadata from the VisitInfo 566 \param calibImage The image whose metadata is to be set 567 \param exposureTime The exposure time for the image 568 \param darkTime The time since the last read (default: exposureTime) 572 darkTime = exposureTime
574 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
575 md = calibImage.getMetadata()
577 afwImage.setVisitInfoMetadata(md, visitInfo)
580 """!Scatter the processing among the nodes 582 We scatter each CCD independently (exposures aren't grouped together), 583 to make full use of all available processors. This necessitates piecing 584 everything back together in the same format as ccdIdLists afterwards. 586 Only the master node executes this method. 588 @param pool Process pool 589 @param ccdIdLists Dict of data identifier lists for each CCD name 590 @return Dict of lists of returned data for each CCD name 592 self.log.info(
"Scatter processing")
595 def process(self, cache, ccdId, outputName="postISRCCD", **kwargs):
596 """!Process a CCD, specified by a data identifier 598 After processing, optionally returns a result (produced by 599 the 'processResult' method) calculated from the processed 600 exposure. These results will be gathered by the master node, 601 and is a means for coordinated scaling of all CCDs for flats, 604 Only slave nodes execute this method. 606 @param cache Process pool cache 607 @param ccdId Data identifier for CCD 608 @param outputName Output dataset name for butler 609 @return result from 'processResult' 612 self.log.warn(
"Null identifier received on %s" % NODE)
615 if self.config.clobber
or not sensorRef.datasetExists(outputName):
616 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
619 except Exception
as e:
620 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
626 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
627 exposure = sensorRef.get(outputName)
631 """Process a single CCD, specified by a data reference 633 Generally, this simply means doing ISR. 635 Only slave nodes execute this method. 637 return self.isr.runDataRef(dataRef).exposure
640 """!Write the processed CCD 642 We need to write these out because we can't hold them all in 645 Only slave nodes execute this method. 647 @param dataRef Data reference 648 @param exposure CCD exposure to write 649 @param outputName Output dataset name for butler. 651 dataRef.put(exposure, outputName)
654 """Extract processing results from a processed exposure 656 This method generates what is gathered by the master node. 657 This can be a background measurement or similar for scaling 658 flat-fields. It must be picklable! 660 Only slave nodes execute this method. 665 """!Determine scaling across CCDs and exposures 667 This is necessary mainly for flats, so as to determine a 668 consistent scaling across the entire focal plane. This 669 implementation is simply a placeholder. 671 Only the master node executes this method. 673 @param ccdIdLists Dict of data identifier lists for each CCD tuple 674 @param data Dict of lists of returned data for each CCD tuple 675 @return dict of Struct(ccdScale: scaling for CCD, 676 expScales: scaling for each exposure 679 self.log.info(
"Scale on %s" % NODE)
680 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
681 for name
in ccdIdLists)
684 """!Scatter the combination of exposures across multiple nodes 686 In this case, we can only scatter across as many nodes as 689 Only the master node executes this method. 691 @param pool Process pool 692 @param outputId Output identifier (exposure part only) 693 @param ccdIdLists Dict of data identifier lists for each CCD name 694 @param scales Dict of structs with scales, for each CCD name 695 @param dict of binned images 697 self.log.info(
"Scatter combination")
698 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for 699 ccdName
in ccdIdLists]
700 images = pool.map(self.
combine, data, outputId)
701 return dict(zip(ccdIdLists.keys(), images))
704 """Get fully-qualified output data identifier 706 We may need to look up keys that aren't in the output dataId. 708 @param ccdName Name tuple for CCD 709 @param butler Data butler 710 @param outputId Data identifier for combined image (exposure part only) 711 @return fully-qualified output dataId 713 fullOutputId = {k: ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
714 fullOutputId.update(outputId)
716 fullOutputId.update(outputId)
720 """!Combine multiple exposures of a particular CCD and write the output 722 Only the slave nodes execute this method. 724 @param cache Process pool cache 725 @param struct Parameters for the combination, which has the following components: 726 * ccdName Name tuple for CCD 727 * ccdIdList List of data identifiers for combination 728 * scales Scales to apply (expScales are scalings for each exposure, 729 ccdScale is final scale for combined image) 730 @param outputId Data identifier for combined image (exposure part only) 731 @return binned calib image 734 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 735 dataId
in struct.ccdIdList]
736 self.log.info(
"Combining %s on %s" % (outputId, NODE))
737 calib = self.combination.
run(dataRefList, expScales=struct.scales.expScales,
738 finalScale=struct.scales.ccdScale)
740 if not hasattr(calib,
"getMetadata"):
741 if hasattr(calib,
"getVariance"):
742 calib = afwImage.makeExposure(calib)
744 calib = afwImage.DecoratedImageF(calib.getImage())
749 struct.ccdIdList, outputId)
753 self.
write(cache.butler, calib, outputId)
755 return afwMath.binImage(calib.getImage(), self.config.binning)
758 """!Record metadata including the inputs and creation details 760 This metadata will go into the FITS header. 762 @param butler Data butler 763 @param calib Combined calib exposure. 764 @param dataIdList List of data identifiers for calibration inputs 765 @param outputId Data identifier for output 767 header = calib.getMetadata()
771 now = time.localtime()
772 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
773 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
775 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
778 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if 780 for i, v
in enumerate(sorted(set(visits))):
781 header.add(
"CALIB_INPUT_%d" % (i,), v)
783 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
784 for key, value
in outputId.items()))
788 """Interpolate over NANs in the combined image 790 NANs can result from masked areas on the CCD. We don't want them getting 791 into our science images, so we replace them with the median of the image. 793 if hasattr(image,
"getMaskedImage"):
795 image = image.getMaskedImage().getImage()
796 if hasattr(image,
"getImage"):
797 image = image.getImage()
798 array = image.getArray()
799 bad = np.isnan(array)
800 array[bad] = np.median(array[np.logical_not(bad)])
802 def write(self, butler, exposure, dataId):
803 """!Write the final combined calib 805 Only the slave nodes execute this method 807 @param butler Data butler 808 @param exposure CCD exposure to write 809 @param dataId Data identifier for output 811 self.log.info(
"Writing %s on %s" % (dataId, NODE))
812 butler.put(exposure, self.
calibName, dataId)
815 """!Create and write an image of the entire camera 817 This is useful for judging the quality or getting an overview of 818 the features of the calib. 820 @param camera Camera object 821 @param dataId Data identifier for output 822 @param calibs Dict mapping CCD detector ID to calib image 824 class ImageSource(object):
825 """Source of images for makeImageFromCamera""" 831 def getCcdImage(self, detector, imageFactory, binSize):
832 detId = detector.getId()
833 if detId
not in self.
images:
834 dims = detector.getBBox().getDimensions()/binSize
835 image = imageFactory(*[int(xx)
for xx
in dims])
838 image = self.
images[detId]
841 image = makeImageFromCamera(camera, imageSource=ImageSource(calibs), imageFactory=afwImage.ImageF,
842 binSize=self.config.binning)
846 """Check that the list of CCD dataIds is consistent 848 @param ccdIdLists Dict of data identifier lists for each CCD name 849 @return Number of exposures, number of CCDs 851 visitIdLists = collections.defaultdict(list)
852 for ccdName
in ccdIdLists:
853 for dataId
in ccdIdLists[ccdName]:
854 visitName =
dictToTuple(dataId, self.config.visitKeys)
855 visitIdLists[visitName].append(dataId)
857 numExps = set(len(expList)
for expList
in ccdIdLists.values())
858 numCcds = set(len(ccdList)
for ccdList
in visitIdLists.values())
860 if len(numExps) != 1
or len(numCcds) != 1:
863 self.log.warn(
"Number of visits for each CCD: %s",
864 {ccdName: len(ccdIdLists[ccdName])
for ccdName
in ccdIdLists})
865 self.log.warn(
"Number of CCDs for each visit: %s",
866 {vv: len(visitIdLists[vv])
for vv
in visitIdLists})
867 raise RuntimeError(
"Inconsistent number of exposures/CCDs")
869 return numExps.pop(), numCcds.pop()
873 """Configuration for bias construction. 875 No changes required compared to the base class, but 876 subclassed for distinction. 881 class BiasTask(CalibTask):
882 """Bias construction""" 883 ConfigClass = BiasConfig
884 _DefaultName =
"bias" 891 """Overrides to apply for bias construction""" 892 config.isr.doBias =
False 893 config.isr.doDark =
False 894 config.isr.doFlat =
False 895 config.isr.doFringe =
False 899 """Configuration for dark construction""" 900 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
901 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
902 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
903 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
904 repair = ConfigurableField(
905 target=RepairTask, doc=
"Task to repair artifacts")
908 CalibConfig.setDefaults(self)
915 The only major difference from the base class is a cosmic-ray 916 identification stage, and dividing each image by the dark time 917 to generate images of the dark rate. 919 ConfigClass = DarkConfig
920 _DefaultName =
"dark" 925 CalibTask.__init__(self, *args, **kwargs)
926 self.makeSubtask(
"repair")
930 """Overrides to apply for dark construction""" 931 config.isr.doDark =
False 932 config.isr.doFlat =
False 933 config.isr.doFringe =
False 936 """Process a single CCD 938 Besides the regular ISR, also masks cosmic-rays and divides each 939 processed image by the dark time to generate images of the dark rate. 940 The dark time is provided by the 'getDarkTime' method. 942 exposure = CalibTask.processSingle(self, sensorRef)
944 if self.config.doRepair:
945 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
946 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
948 self.repair.
run(exposure, keepCRs=
False)
949 if self.config.crGrow > 0:
950 mask = exposure.getMaskedImage().getMask().clone()
951 mask &= mask.getPlaneBitMask(
"CR")
952 fpSet = afwDet.FootprintSet(
953 mask, afwDet.Threshold(0.5))
954 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
955 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
957 mi = exposure.getMaskedImage()
962 """Retrieve the dark time for an exposure""" 963 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
964 if not np.isfinite(darkTime):
965 raise RuntimeError(
"Non-finite darkTime")
970 """Configuration for flat construction""" 971 iterations = Field(dtype=int, default=10,
972 doc=
"Number of iterations for scale determination")
973 stats = ConfigurableField(target=CalibStatsTask,
974 doc=
"Background statistics configuration")
980 The principal change from the base class involves gathering the background 981 values from each image and using them to determine the scalings for the final 984 ConfigClass = FlatConfig
985 _DefaultName =
"flat" 990 """Overrides for flat construction""" 991 config.isr.doFlat =
False 992 config.isr.doFringe =
False 995 CalibTask.__init__(self, *args, **kwargs)
996 self.makeSubtask(
"stats")
999 return self.stats.
run(exposure)
1002 """Determine the scalings for the final combination 1004 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling 1005 of one CCD to all the others in an exposure, and E_j is the scaling 1006 of the exposure. We convert everything to logarithms so we can work 1007 with a linear system. We determine the C_i and E_j from B_ij by iteration, 1008 under the additional constraint that the average CCD scale is unity. 1010 This algorithm comes from Eugene Magnier and Pan-STARRS. 1012 assert len(ccdIdLists.values()) > 0,
"No successful CCDs" 1013 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
1015 lengths) == 1,
"Number of successful exposures for each CCD differs" 1016 assert tuple(lengths)[0] > 0,
"No successful exposures" 1018 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
1019 bgMatrix = np.array([[0.0] * len(expList)
1020 for expList
in ccdIdLists.values()])
1021 for name
in ccdIdLists:
1024 d
if d
is not None else np.nan
for d
in data[name]]
1026 numpyPrint = np.get_printoptions()
1027 np.set_printoptions(threshold=np.inf)
1028 self.log.info(
"Input backgrounds: %s" % bgMatrix)
1031 numCcds = len(ccdIdLists)
1032 numExps = bgMatrix.shape[1]
1034 bgMatrix = np.log(bgMatrix)
1035 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
1037 compScales = np.zeros(numCcds)
1038 expScales = np.array(
1039 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
1041 for iterate
in range(self.config.iterations):
1042 compScales = np.array(
1043 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
1044 expScales = np.array(
1045 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
1047 avgScale = np.average(np.exp(compScales))
1048 compScales -= np.log(avgScale)
1049 self.log.debug(
"Iteration %d exposure scales: %s",
1050 iterate, np.exp(expScales))
1051 self.log.debug(
"Iteration %d component scales: %s",
1052 iterate, np.exp(compScales))
1054 expScales = np.array(
1055 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
1057 if np.any(np.isnan(expScales)):
1058 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
1059 (bgMatrix, expScales))
1061 expScales = np.exp(expScales)
1062 compScales = np.exp(compScales)
1064 self.log.info(
"Exposure scales: %s" % expScales)
1065 self.log.info(
"Component relative scaling: %s" % compScales)
1066 np.set_printoptions(**numpyPrint)
1068 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
1069 for ccdName
in ccdIdLists)
1073 """Configuration for fringe construction""" 1074 stats = ConfigurableField(target=CalibStatsTask,
1075 doc=
"Background statistics configuration")
1076 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1077 doc=
"Background configuration")
1078 detection = ConfigurableField(
1079 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1080 detectSigma = Field(dtype=float, default=1.0,
1081 doc=
"Detection PSF gaussian sigma")
1085 """Fringe construction task 1087 The principal change from the base class is that the images are 1088 background-subtracted and rescaled by the background. 1090 XXX This is probably not right for a straight-up combination, as we 1091 are currently doing, since the fringe amplitudes need not scale with 1094 XXX Would like to have this do PCA and generate multiple images, but 1095 that will take a bit of work with the persistence code. 1097 ConfigClass = FringeConfig
1098 _DefaultName =
"fringe" 1099 calibName =
"fringe" 1103 """Overrides for fringe construction""" 1104 config.isr.doFringe =
False 1107 CalibTask.__init__(self, *args, **kwargs)
1108 self.makeSubtask(
"detection")
1109 self.makeSubtask(
"stats")
1110 self.makeSubtask(
"subtractBackground")
1113 """Subtract the background and normalise by the background level""" 1114 exposure = CalibTask.processSingle(self, sensorRef)
1115 bgLevel = self.stats.
run(exposure)
1116 self.subtractBackground.
run(exposure)
1117 mi = exposure.getMaskedImage()
1119 footprintSets = self.detection.detectFootprints(
1120 exposure, sigma=self.config.detectSigma)
1121 mask = exposure.getMaskedImage().getMask()
1122 detected = 1 << mask.addMaskPlane(
"DETECTED")
1123 for fpSet
in (footprintSets.positive, footprintSets.negative):
1124 if fpSet
is not None:
1125 afwDet.setMaskFromFootprintList(
1126 mask, fpSet.getFootprints(), detected)
1131 """Configuration for sky frame construction""" 1132 detection = ConfigurableField(target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
1133 detectSigma = Field(dtype=float, default=2.0, doc=
"Detection PSF gaussian sigma")
1134 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
1135 doc=
"Regular-scale background configuration, for object detection")
1136 largeScaleBackground = ConfigField(dtype=FocalPlaneBackgroundConfig,
1137 doc=
"Large-scale background configuration")
1138 sky = ConfigurableField(target=SkyMeasurementTask, doc=
"Sky measurement")
1139 maskThresh = Field(dtype=float, default=3.0, doc=
"k-sigma threshold for masking pixels")
1140 mask = ListField(dtype=str, default=[
"BAD",
"SAT",
"DETECTED",
"NO_DATA"],
1141 doc=
"Mask planes to consider as contaminated")
1145 """Task for sky frame construction 1147 The sky frame is a (relatively) small-scale background 1148 model, the response of the camera to the sky. 1150 To construct, we first remove a large-scale background (e.g., caused 1151 by moonlight) which may vary from image to image. Then we construct a 1152 model of the sky, which is essentially a binned version of the image 1153 (important configuration parameters: sky.background.[xy]BinSize). 1154 It is these models which are coadded to yield the sky frame. 1156 ConfigClass = SkyConfig
1157 _DefaultName =
"sky" 1161 CalibTask.__init__(self, *args, **kwargs)
1162 self.makeSubtask(
"detection")
1163 self.makeSubtask(
"subtractBackground")
1164 self.makeSubtask(
"sky")
1167 """!Scatter the processing among the nodes 1169 Only the master node executes this method, assigning work to the 1172 We measure and subtract off a large-scale background model across 1173 all CCDs, which requires a scatter/gather. Then we process the 1174 individual CCDs, subtracting the large-scale background model and 1175 the residual background model measured. These residuals will be 1176 combined for the sky frame. 1178 @param pool Process pool 1179 @param ccdIdLists Dict of data identifier lists for each CCD name 1180 @return Dict of lists of returned data for each CCD name 1182 self.log.info(
"Scatter processing")
1184 numExps = set(len(expList)
for expList
in ccdIdLists.values()).pop()
1192 for exp
in range(numExps):
1193 bgModels = [bgModelList[ccdName][exp]
for ccdName
in ccdIdLists]
1194 visit = set(tuple(ccdIdLists[ccdName][exp][key]
for key
in sorted(self.config.visitKeys))
for 1195 ccdName
in ccdIdLists)
1196 assert len(visit) == 1
1198 bgModel = bgModels[0]
1199 for bg
in bgModels[1:]:
1201 self.log.info(
"Background model min/max for visit %s: %f %f", visit,
1202 np.min(bgModel.getStatsImage().getArray()),
1203 np.max(bgModel.getStatsImage().getArray()))
1204 backgrounds[visit] = bgModel
1205 scales[visit] = np.median(bgModel.getStatsImage().getArray())
1207 return mapToMatrix(pool, self.
process, ccdIdLists, backgrounds=backgrounds, scales=scales)
1210 """!Measure background model for CCD 1212 This method is executed by the slaves. 1214 The background models for all CCDs in an exposure will be 1215 combined to form a full focal-plane background model. 1217 @param cache Process pool cache 1218 @param dataId Data identifier 1219 @return Bcakground model 1226 config = self.config.largeScaleBackground
1227 camera = dataRef.get(
"camera")
1228 bgModel = FocalPlaneBackground.fromCamera(config, camera)
1229 bgModel.addCcd(exposure)
1233 """!Process a single CCD for the background 1235 This method is executed by the slaves. 1237 Because we're interested in the background, we detect and mask astrophysical 1238 sources, and pixels above the noise level. 1240 @param dataRef Data reference for CCD. 1241 @return processed exposure 1243 if not self.config.clobber
and dataRef.datasetExists(
"postISRCCD"):
1244 return dataRef.get(
"postISRCCD")
1245 exposure = CalibTask.processSingle(self, dataRef)
1248 bgTemp = self.subtractBackground.
run(exposure).background
1249 footprints = self.detection.detectFootprints(exposure, sigma=self.config.detectSigma)
1250 image = exposure.getMaskedImage()
1251 if footprints.background
is not None:
1252 image += footprints.background.getImageF()
1255 variance = image.getVariance()
1256 noise = np.sqrt(np.median(variance.getArray()))
1257 isHigh = image.getImage().getArray() > self.config.maskThresh*noise
1258 image.getMask().getArray()[isHigh] |= image.getMask().getPlaneBitMask(
"DETECTED")
1261 image += bgTemp.getImage()
1264 maskVal = image.getMask().getPlaneBitMask(self.config.mask)
1265 isBad = image.getMask().getArray() & maskVal > 0
1266 bgLevel = np.median(image.getImage().getArray()[~isBad])
1267 image.getImage().getArray()[isBad] = bgLevel
1268 dataRef.put(exposure,
"postISRCCD")
1272 """Process a single CCD, specified by a data reference 1274 We subtract the appropriate focal plane background model, 1275 divide by the appropriate scale and measure the background. 1277 Only slave nodes execute this method. 1279 @param dataRef Data reference for single CCD 1280 @param backgrounds Background model for each visit 1281 @param scales Scales for each visit 1282 @return Processed exposure 1284 visit = tuple(dataRef.dataId[key]
for key
in sorted(self.config.visitKeys))
1285 exposure = dataRef.get(
"postISRCCD", immediate=
True)
1286 image = exposure.getMaskedImage()
1287 detector = exposure.getDetector()
1288 bbox = image.getBBox()
1290 bgModel = backgrounds[visit]
1291 bg = bgModel.toCcdBackground(detector, bbox)
1292 image -= bg.getImage()
1293 image /= scales[visit]
1296 dataRef.put(bg,
"icExpBackground")
1300 """!Combine multiple background models of a particular CCD and write the output 1302 Only the slave nodes execute this method. 1304 @param cache Process pool cache 1305 @param struct Parameters for the combination, which has the following components: 1306 * ccdName Name tuple for CCD 1307 * ccdIdList List of data identifiers for combination 1308 @param outputId Data identifier for combined image (exposure part only) 1309 @return binned calib image 1312 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 1313 dataId
in struct.ccdIdList]
1314 self.log.info(
"Combining %s on %s" % (outputId, NODE))
1315 bgList = [dataRef.get(
"icExpBackground", immediate=
True).clone()
for dataRef
in dataRefList]
1317 bgExp = self.sky.averageBackgrounds(bgList)
1320 cache.butler.put(bgExp,
"sky", outputId)
1321 return afwMath.binImage(self.sky.exposureToBackground(bgExp).getImage(), self.config.binning)
def applyOverrides(cls, config)
def getFilter(self, butler, dataId)
def __init__(self, args, kwargs)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def __init__(self, args, kwargs)
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def processResult(self, exposure)
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def __init__(self, calibName, args, kwargs)
def applyScale(self, exposure, scale=None)
def processSingle(self, sensorRef)
def applyOverrides(cls, config)
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def processSingle(self, sensorRef)
def getFullyQualifiedOutputId(self, ccdName, butler, outputId)
def __call__(self, parser, namespace, values, option_string)
def processSingle(self, dataRef)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def interpolateNans(self, image)
def getDataRef(butler, dataId, datasetType="raw")
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def scale(self, ccdIdLists, data)
def process(self, cache, ccdId, outputName="postISRCCD", kwargs)
Process a CCD, specified by a data identifier.
def parse_args(self, args, kwargs)
def processSingle(self, dataRef, backgrounds, scales)
def makeCameraImage(self, camera, dataId, calibs)
Create and write an image of the entire camera.
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def getDarkTime(self, exposure)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def checkCcdIdLists(self, ccdIdLists)
def write(self, butler, exposure, dataId)
Write the final combined calib.
def combine(self, cache, struct, outputId)
Combine multiple background models of a particular CCD and write the output.
def measureBackground(self, cache, dataId)
Measure background model for CCD.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def __init__(self, args, kwargs)
def __init__(self, args, kwargs)
def applyOverrides(cls, config)
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def processResult(self, exposure)
def mapToMatrix(pool, func, ccdIdLists, args, kwargs)
def __init__(self, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
def getTargetList(parsedCmd, kwargs)
def __init__(self, args, kwargs)
Base class for constructing calibs.
def applyOverrides(cls, config)
def processSingleBackground(self, dataRef)
Process a single CCD for the background.
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)