1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 from lsst.afw.image
import VisitInfo
20 import lsst.meas.algorithms
as measAlg
21 from lsst.pipe.tasks.repair
import RepairTask
22 from lsst.ip.isr
import IsrTask
24 from lsst.ctrl.pool.parallel
import BatchPoolTask
25 from lsst.ctrl.pool.pool
import Pool, NODE
27 from .checksum
import checksum
28 from .utils
import getDataRef
32 """Parameters controlling the measurement of background statistics"""
33 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34 default=int(afwMath.MEANCLIP))
35 clip = Field(doc=
"Clipping threshold for background",
36 dtype=float, default=3.0)
37 nIter = Field(doc=
"Clipping iterations for background",
39 mask = ListField(doc=
"Mask planes to reject",
40 dtype=str, default=[
"DETECTED",
"BAD"])
44 """Measure statistics on the background
46 This can be useful for scaling the background, e.g., for flats and fringe frames.
48 ConfigClass = CalibStatsConfig
50 def run(self, exposureOrImage):
51 """!Measure a particular statistic on an image (of some sort).
53 @param exposureOrImage Exposure, MaskedImage or Image.
54 @return Value of desired statistic
56 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57 afwImage.Mask.getPlaneBitMask(self.config.mask))
59 image = exposureOrImage.getMaskedImage()
62 image = exposureOrImage.getImage()
64 image = exposureOrImage
66 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
70 """Configuration for combining calib images"""
71 rows = Field(doc=
"Number of rows to read at a time",
72 dtype=int, default=512)
73 mask = ListField(doc=
"Mask planes to respect", dtype=str,
74 default=[
"SAT",
"DETECTED",
"INTRP"])
75 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
76 default=int(afwMath.MEANCLIP))
77 clip = Field(doc=
"Clipping threshold for combination",
78 dtype=float, default=3.0)
79 nIter = Field(doc=
"Clipping iterations for combination",
81 stats = ConfigurableField(target=CalibStatsTask,
82 doc=
"Background statistics configuration")
86 """Task to combine calib images"""
87 ConfigClass = CalibCombineConfig
90 Task.__init__(self, *args, **kwargs)
91 self.makeSubtask(
"stats")
93 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94 """!Combine calib images for a single sensor
96 @param sensorRefList List of data references to combine (for a single sensor)
97 @param expScales List of scales to apply for each exposure
98 @param finalScale Desired scale for final combined image
99 @param inputName Data set name for inputs
100 @return combined image
103 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104 afwImage.Mask.getPlaneBitMask(self.config.mask))
107 combined = afwImage.MaskedImageF(width, height)
108 numImages = len(sensorRefList)
109 imageList = [
None]*numImages
110 for start
in range(0, height, self.config.rows):
111 rows = min(self.config.rows, height - start)
112 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113 afwGeom.Extent2I(width, rows))
114 subCombined = combined.Factory(combined, box)
116 for i, sensorRef
in enumerate(sensorRefList):
117 if sensorRef
is None:
120 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
121 if expScales
is not None:
123 imageList[i] = exposure.getMaskedImage()
125 self.
combine(subCombined, imageList, stats)
127 if finalScale
is not None:
128 background = self.stats.run(combined)
129 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
130 (NODE, background, finalScale))
131 combined *= finalScale / background
136 """Get dimensions of the inputs"""
138 for sensorRef
in sensorRefList:
139 if sensorRef
is None:
141 md = sensorRef.get(inputName +
"_md")
142 dimList.append(afwGeom.Extent2I(
143 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
147 """Apply scale to input exposure
149 This implementation applies a flux scaling: the input exposure is
150 divided by the provided scale.
152 if scale
is not None:
153 mi = exposure.getMaskedImage()
157 """!Combine multiple images
159 @param target Target image to receive the combined pixels
160 @param imageList List of input images
161 @param stats Statistics control
163 images = [img
for img
in imageList
if img
is not None]
164 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
168 """Determine a consistent size, given a list of image sizes"""
169 dim = set((w, h)
for w, h
in dimList)
172 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
177 """!Return a tuple of specific values from a dict
179 This provides a hashable representation of the dict from certain keywords.
180 This can be useful for creating e.g., a tuple of the values in the DataId
181 that identify the CCD.
183 @param dict_ dict to parse
184 @param keys keys to extract (order is important)
185 @return tuple of values
187 return tuple(dict_[k]
for k
in keys)
191 """!Determine a list of CCDs from exposure references
193 This essentially inverts the exposure-level references (which
194 provides a list of CCDs for each exposure), by providing
195 a dataId list for each CCD. Consider an input list of exposures
196 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
199 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
201 This is a dict whose keys are tuples of the identifying values of a
202 CCD (usually just the CCD number) and the values are lists of dataIds
203 for that CCD in each exposure. A missing dataId is given the value
206 @param expRefList List of data references for exposures
207 @param level Level for the butler to generate CCDs
208 @param ccdKeys DataId keywords that identify a CCD
209 @return dict of data identifier lists for each CCD
211 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
212 level)]
for expRef
in expRefList]
215 ccdKeys = set(ccdKeys)
217 for ccdIdList
in expIdList:
218 for ccdId
in ccdIdList:
225 for n, ccdIdList
in enumerate(expIdList):
226 for ccdId
in ccdIdList:
228 if name
not in ccdLists:
230 ccdLists[name].append(ccdId)
236 """Split name=value pairs and put the result in a dict"""
238 def __call__(self, parser, namespace, values, option_string):
239 output = getattr(namespace, self.dest, {})
240 for nameValue
in values:
241 name, sep, valueStr = nameValue.partition(
"=")
243 parser.error(
"%s value %s must be in form name=value" %
244 (option_string, nameValue))
245 output[name] = valueStr
246 setattr(namespace, self.dest, output)
250 """ArgumentParser for calibration construction"""
253 """Add a --calibId argument to the standard pipe_base argument parser"""
254 ArgumentParser.__init__(self, *args, **kwargs)
256 self.add_id_argument(
"--id", datasetType=
"raw",
257 help=
"input identifiers, e.g., --id visit=123 ccd=4")
258 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
259 help=
"identifiers for calib, e.g., --calibId version=1",
260 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
265 Checks that the "--calibId" provided works.
267 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
269 keys = namespace.butler.getKeys(self.
calibName)
271 for name, value
in namespace.calibId.items():
274 "%s is not a relevant calib identifier key (%s)" % (name, keys))
275 parsed[name] = keys[name](value)
276 namespace.calibId = parsed
282 """Configuration for constructing calibs"""
283 clobber = Field(dtype=bool, default=
True,
284 doc=
"Clobber existing processed images?")
285 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
286 dateObs = Field(dtype=str, default=
"dateObs",
287 doc=
"Key for observation date in exposure registry")
288 dateCalib = Field(dtype=str, default=
"calibDate",
289 doc=
"Key for calib date in calib registry")
290 filter = Field(dtype=str, default=
"filter",
291 doc=
"Key for filter name in exposure/calib registries")
292 combination = ConfigurableField(
293 target=CalibCombineTask, doc=
"Calib combination configuration")
294 ccdKeys = ListField(dtype=str, default=[
295 "ccd"], doc=
"DataId keywords specifying a CCD")
296 visitKeys = ListField(dtype=str, default=[
297 "visit"], doc=
"DataId keywords specifying a visit")
298 calibKeys = ListField(dtype=str, default=[],
299 doc=
"DataId keywords specifying a calibration")
302 self.isr.doWrite =
False
306 """Get parsed values into the CalibTask.run"""
309 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
312 """Call the Task with the kwargs from getTargetList"""
313 task = self.TaskClass(config=self.config, log=self.log)
316 result = task.run(**args)
319 result = task.run(**args)
320 except Exception
as e:
323 task.log.fatal(
"Failed: %s" % e)
324 traceback.print_exc(file=sys.stderr)
326 if self.doReturnResults:
328 exitStatus=exitStatus,
330 metadata=task.metadata,
335 exitStatus=exitStatus,
339 """!Base class for constructing calibs.
341 This should be subclassed for each of the required calib types.
342 The subclass should be sure to define the following class variables:
343 * _DefaultName: default name of the task, used by CmdLineTask
344 * calibName: name of the calibration data set in the butler
345 The subclass may optionally set:
346 * filterName: filter name to give the resultant calib
348 ConfigClass = CalibConfig
349 RunnerClass = CalibTaskRunner
356 BatchPoolTask.__init__(self, *args, **kwargs)
357 self.makeSubtask(
"isr")
358 self.makeSubtask(
"combination")
362 numCcds = len(parsedCmd.butler.get(
"camera"))
363 numExps = len(cls.RunnerClass.getTargetList(
364 parsedCmd)[0][
'expRefList'])
365 numCycles = int(numCcds/float(numCores) + 0.5)
366 return time*numExps*numCycles
369 def _makeArgumentParser(cls, *args, **kwargs):
370 kwargs.pop(
"doBatch",
False)
371 return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
373 def run(self, expRefList, butler, calibId):
374 """!Construct a calib from a list of exposure references
376 This is the entry point, called by the TaskRunner.__call__
378 Only the master node executes this method.
380 @param expRefList List of data references at the exposure level
381 @param butler Data butler
382 @param calibId Identifier dict for calib
384 for expRef
in expRefList:
385 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
389 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
392 outputIdItemList = list(outputId.items())
393 for ccdName
in ccdIdLists:
394 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
395 dataId.update(outputIdItemList)
397 dataId.update(outputIdItemList)
400 butler.get(self.
calibName +
"_filename", dataId)
401 except Exception
as e:
403 "Unable to determine output filename \"%s_filename\" from %s: %s" %
407 pool.storeSet(butler=butler)
413 scales = self.
scale(ccdIdLists, data)
419 """!Generate the data identifier for the output calib
421 The mean date and the common filter are included, using keywords
422 from the configuration. The CCD-specific part is not included
423 in the data identifier.
425 @param expRefList List of data references at exposure level
426 @param calibId Data identifier elements for the calib provided by the user
427 @return data identifier
431 for expRef
in expRefList:
432 butler = expRef.getButler()
433 dataId = expRef.dataId
435 midTime += self.
getMjd(butler, dataId)
438 if filterName
is None:
439 filterName = thisFilter
440 elif filterName != thisFilter:
441 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
442 dataId, thisFilter, filterName))
444 midTime /= len(expRefList)
445 date = str(dafBase.DateTime(
446 midTime, dafBase.DateTime.MJD).toPython().date())
448 outputId = {self.config.filter: filterName,
449 self.config.dateCalib: date}
450 outputId.update(calibId)
453 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
454 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
455 if self.config.dateObs
in dataId:
456 dateObs = dataId[self.config.dateObs]
458 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
459 if "T" not in dateObs:
460 dateObs = dateObs +
"T12:00:00.0Z"
461 elif not dateObs.endswith(
"Z"):
464 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
467 """Determine the filter from a data identifier"""
468 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
472 if calibName
is None:
475 if missingKeys
is None:
476 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
478 for k
in missingKeys:
480 v = butler.queryMetadata(
'raw', [k], dataId)
481 except Exception
as e:
490 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
493 """!Update the metadata from the VisitInfo
495 \param calibImage The image whose metadata is to be set
496 \param exposureTime The exposure time for the image
497 \param darkTime The time since the last read (default: exposureTime)
501 darkTime = exposureTime
503 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
504 md = calibImage.getMetadata()
506 afwImage.setVisitInfoMetadata(md, visitInfo)
509 """!Scatter the processing among the nodes
511 We scatter each CCD independently (exposures aren't grouped together),
512 to make full use of all available processors. This necessitates piecing
513 everything back together in the same format as ccdIdLists afterwards.
515 Only the master node executes this method.
517 @param pool Process pool
518 @param ccdIdLists Dict of data identifier lists for each CCD name
519 @return Dict of lists of returned data for each CCD name
521 dataIdList = sum(ccdIdLists.values(), [])
522 self.log.info(
"Scatter processing")
524 resultList = pool.map(self.
process, dataIdList)
527 data = dict((ccdName, [
None] * len(expList))
528 for ccdName, expList
in ccdIdLists.items())
529 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
530 for expNum, dataId
in enumerate(expList)]
531 for ccdName, expList
in ccdIdLists.items()], []))
532 for dataId, result
in zip(dataIdList, resultList):
535 ccdName, expNum = indices[tuple(dataId.values())]
536 data[ccdName][expNum] = result
540 def process(self, cache, ccdId, outputName="postISRCCD"):
541 """!Process a CCD, specified by a data identifier
543 After processing, optionally returns a result (produced by
544 the 'processResult' method) calculated from the processed
545 exposure. These results will be gathered by the master node,
546 and is a means for coordinated scaling of all CCDs for flats,
549 Only slave nodes execute this method.
551 @param cache Process pool cache
552 @param ccdId Data identifier for CCD
553 @param outputName Output dataset name for butler
554 @return result from 'processResult'
557 self.log.warn(
"Null identifier received on %s" % NODE)
560 if self.config.clobber
or not sensorRef.datasetExists(outputName):
561 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
564 except Exception
as e:
565 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
571 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
572 exposure = sensorRef.get(outputName, immediate=
True)
576 """Process a single CCD, specified by a data reference
578 Generally, this simply means doing ISR.
580 Only slave nodes execute this method.
582 return self.isr.runDataRef(dataRef).exposure
585 """!Write the processed CCD
587 We need to write these out because we can't hold them all in
590 Only slave nodes execute this method.
592 @param dataRef Data reference
593 @param exposure CCD exposure to write
594 @param outputName Output dataset name for butler.
596 dataRef.put(exposure, outputName)
599 """Extract processing results from a processed exposure
601 This method generates what is gathered by the master node.
602 This can be a background measurement or similar for scaling
603 flat-fields. It must be picklable!
605 Only slave nodes execute this method.
610 """!Determine scaling across CCDs and exposures
612 This is necessary mainly for flats, so as to determine a
613 consistent scaling across the entire focal plane. This
614 implementation is simply a placeholder.
616 Only the master node executes this method.
618 @param ccdIdLists Dict of data identifier lists for each CCD tuple
619 @param data Dict of lists of returned data for each CCD tuple
620 @return dict of Struct(ccdScale: scaling for CCD,
621 expScales: scaling for each exposure
624 self.log.info(
"Scale on %s" % NODE)
625 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
626 for name
in ccdIdLists)
629 """!Scatter the combination of exposures across multiple nodes
631 In this case, we can only scatter across as many nodes as
634 Only the master node executes this method.
636 @param pool Process pool
637 @param outputId Output identifier (exposure part only)
638 @param ccdIdLists Dict of data identifier lists for each CCD name
639 @param scales Dict of structs with scales, for each CCD name
641 self.log.info(
"Scatter combination")
642 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for
643 ccdName
in ccdIdLists]
644 pool.map(self.
combine, data, outputId)
647 """!Combine multiple exposures of a particular CCD and write the output
649 Only the slave nodes execute this method.
651 @param cache Process pool cache
652 @param struct Parameters for the combination, which has the following components:
653 * ccdName Name tuple for CCD
654 * ccdIdList List of data identifiers for combination
655 * scales Scales to apply (expScales are scalings for each exposure,
656 ccdScale is final scale for combined image)
657 @param outputId Data identifier for combined image (exposure part only)
660 fullOutputId = {k: struct.ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
661 fullOutputId.update(outputId)
663 fullOutputId.update(outputId)
664 outputId = fullOutputId
667 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for
668 dataId
in struct.ccdIdList]
669 self.log.info(
"Combining %s on %s" % (outputId, NODE))
670 calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
671 finalScale=struct.scales.ccdScale)
673 if not hasattr(calib,
"getMetadata"):
674 if hasattr(calib,
"getVariance"):
675 calib = afwImage.makeExposure(calib)
677 calib = afwImage.DecoratedImageF(calib.getImage())
682 struct.ccdIdList, outputId)
686 self.
write(cache.butler, calib, outputId)
689 """!Record metadata including the inputs and creation details
691 This metadata will go into the FITS header.
693 @param butler Data butler
694 @param calib Combined calib exposure.
695 @param dataIdList List of data identifiers for calibration inputs
696 @param outputId Data identifier for output
698 header = calib.getMetadata()
702 now = time.localtime()
703 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
704 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
706 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
709 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if
711 for i, v
in enumerate(sorted(set(visits))):
712 header.add(
"CALIB_INPUT_%d" % (i,), v)
714 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
715 for key, value
in outputId.items()))
719 """Interpolate over NANs in the combined image
721 NANs can result from masked areas on the CCD. We don't want them getting
722 into our science images, so we replace them with the median of the image.
724 if hasattr(image,
"getMaskedImage"):
726 image = image.getMaskedImage().getImage()
727 if hasattr(image,
"getImage"):
728 image = image.getImage()
729 array = image.getArray()
730 bad = np.isnan(array)
731 array[bad] = np.median(array[np.logical_not(bad)])
733 def write(self, butler, exposure, dataId):
734 """!Write the final combined calib
736 Only the slave nodes execute this method
738 @param butler Data butler
739 @param exposure CCD exposure to write
740 @param dataId Data identifier for output
742 self.log.info(
"Writing %s on %s" % (dataId, NODE))
743 butler.put(exposure, self.
calibName, dataId)
747 """Configuration for bias construction.
749 No changes required compared to the base class, but
750 subclassed for distinction.
755 class BiasTask(CalibTask):
756 """Bias construction"""
757 ConfigClass = BiasConfig
758 _DefaultName =
"bias"
765 """Overrides to apply for bias construction"""
766 config.isr.doBias =
False
767 config.isr.doDark =
False
768 config.isr.doFlat =
False
769 config.isr.doFringe =
False
773 """Configuration for dark construction"""
774 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
775 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
776 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
777 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
778 repair = ConfigurableField(
779 target=RepairTask, doc=
"Task to repair artifacts")
782 CalibConfig.setDefaults(self)
783 self.combination.mask.append(
"CR")
789 The only major difference from the base class is a cosmic-ray
790 identification stage, and dividing each image by the dark time
791 to generate images of the dark rate.
793 ConfigClass = DarkConfig
794 _DefaultName =
"dark"
799 CalibTask.__init__(self, *args, **kwargs)
800 self.makeSubtask(
"repair")
804 """Overrides to apply for dark construction"""
805 config.isr.doDark =
False
806 config.isr.doFlat =
False
807 config.isr.doFringe =
False
810 """Process a single CCD
812 Besides the regular ISR, also masks cosmic-rays and divides each
813 processed image by the dark time to generate images of the dark rate.
814 The dark time is provided by the 'getDarkTime' method.
816 exposure = CalibTask.processSingle(self, sensorRef)
818 if self.config.doRepair:
819 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
820 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
822 self.repair.run(exposure, keepCRs=
False)
823 if self.config.crGrow > 0:
824 mask = exposure.getMaskedImage().getMask().clone()
825 mask &= mask.getPlaneBitMask(
"CR")
826 fpSet = afwDet.FootprintSet(
827 mask, afwDet.Threshold(0.5))
828 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
829 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
831 mi = exposure.getMaskedImage()
836 """Retrieve the dark time for an exposure"""
837 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
838 if not np.isfinite(darkTime):
839 raise RuntimeError(
"Non-finite darkTime")
844 """Configuration for flat construction"""
845 iterations = Field(dtype=int, default=10,
846 doc=
"Number of iterations for scale determination")
847 stats = ConfigurableField(target=CalibStatsTask,
848 doc=
"Background statistics configuration")
854 The principal change from the base class involves gathering the background
855 values from each image and using them to determine the scalings for the final
858 ConfigClass = FlatConfig
859 _DefaultName =
"flat"
864 """Overrides for flat construction"""
865 config.isr.doFlat =
False
866 config.isr.doFringe =
False
869 CalibTask.__init__(self, *args, **kwargs)
870 self.makeSubtask(
"stats")
873 return self.stats.run(exposure)
876 """Determine the scalings for the final combination
878 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
879 of one CCD to all the others in an exposure, and E_j is the scaling
880 of the exposure. We convert everything to logarithms so we can work
881 with a linear system. We determine the C_i and E_j from B_ij by iteration,
882 under the additional constraint that the average CCD scale is unity.
884 This algorithm comes from Eugene Magnier and Pan-STARRS.
886 assert len(ccdIdLists.values()) > 0,
"No successful CCDs"
887 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
889 lengths) == 1,
"Number of successful exposures for each CCD differs"
890 assert tuple(lengths)[0] > 0,
"No successful exposures"
892 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
893 bgMatrix = np.array([[0.0] * len(expList)
894 for expList
in ccdIdLists.values()])
895 for name
in ccdIdLists:
898 d
if d
is not None else np.nan
for d
in data[name]]
900 numpyPrint = np.get_printoptions()
901 np.set_printoptions(threshold=
'nan')
902 self.log.info(
"Input backgrounds: %s" % bgMatrix)
905 numCcds = len(ccdIdLists)
906 numExps = bgMatrix.shape[1]
908 bgMatrix = np.log(bgMatrix)
909 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
911 compScales = np.zeros(numCcds)
912 expScales = np.array(
913 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
915 for iterate
in range(self.config.iterations):
916 compScales = np.array(
917 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
918 expScales = np.array(
919 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
921 avgScale = np.average(np.exp(compScales))
922 compScales -= np.log(avgScale)
923 self.log.debug(
"Iteration %d exposure scales: %s",
924 iterate, np.exp(expScales))
925 self.log.debug(
"Iteration %d component scales: %s",
926 iterate, np.exp(compScales))
928 expScales = np.array(
929 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
931 if np.any(np.isnan(expScales)):
932 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
933 (bgMatrix, expScales))
935 expScales = np.exp(expScales)
936 compScales = np.exp(compScales)
938 self.log.info(
"Exposure scales: %s" % expScales)
939 self.log.info(
"Component relative scaling: %s" % compScales)
940 np.set_printoptions(**numpyPrint)
942 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
943 for ccdName
in ccdIdLists)
947 """Configuration for fringe construction"""
948 stats = ConfigurableField(target=CalibStatsTask,
949 doc=
"Background statistics configuration")
950 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
951 doc=
"Background configuration")
952 detection = ConfigurableField(
953 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
954 detectSigma = Field(dtype=float, default=1.0,
955 doc=
"Detection PSF gaussian sigma")
959 """Fringe construction task
961 The principal change from the base class is that the images are
962 background-subtracted and rescaled by the background.
964 XXX This is probably not right for a straight-up combination, as we
965 are currently doing, since the fringe amplitudes need not scale with
968 XXX Would like to have this do PCA and generate multiple images, but
969 that will take a bit of work with the persistence code.
971 ConfigClass = FringeConfig
972 _DefaultName =
"fringe"
977 """Overrides for fringe construction"""
978 config.isr.doFringe =
False
981 CalibTask.__init__(self, *args, **kwargs)
982 self.makeSubtask(
"detection")
983 self.makeSubtask(
"stats")
984 self.makeSubtask(
"subtractBackground")
987 """Subtract the background and normalise by the background level"""
988 exposure = CalibTask.processSingle(self, sensorRef)
989 bgLevel = self.stats.run(exposure)
990 self.subtractBackground.run(exposure)
991 mi = exposure.getMaskedImage()
993 footprintSets = self.detection.detectFootprints(
994 exposure, sigma=self.config.detectSigma)
995 mask = exposure.getMaskedImage().getMask()
996 detected = 1 << mask.addMaskPlane(
"DETECTED")
997 for fpSet
in (footprintSets.positive, footprintSets.negative):
998 if fpSet
is not None:
999 afwDet.setMaskFromFootprintList(
1000 mask, fpSet.getFootprints(), detected)
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
def recordCalibInputs
Record metadata including the inputs and creation details.
def combine
Combine multiple images.
def updateMetadata
Update the metadata from the VisitInfo.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.