1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 from lsst.afw.image
import VisitInfo
20 import lsst.meas.algorithms
as measAlg
21 from lsst.pipe.tasks.repair
import RepairTask
22 from lsst.ip.isr
import IsrTask
24 from lsst.ctrl.pool.parallel
import BatchPoolTask
25 from lsst.ctrl.pool.pool
import Pool, NODE
27 from .checksum
import checksum
28 from .utils
import getDataRef
32 """Parameters controlling the measurement of background statistics"""
33 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34 default=int(afwMath.MEANCLIP))
35 clip = Field(doc=
"Clipping threshold for background",
36 dtype=float, default=3.0)
37 nIter = Field(doc=
"Clipping iterations for background",
39 mask = ListField(doc=
"Mask planes to reject",
40 dtype=str, default=[
"DETECTED",
"BAD"])
44 """Measure statistics on the background
46 This can be useful for scaling the background, e.g., for flats and fringe frames.
48 ConfigClass = CalibStatsConfig
50 def run(self, exposureOrImage):
51 """!Measure a particular statistic on an image (of some sort).
53 @param exposureOrImage Exposure, MaskedImage or Image.
54 @return Value of desired statistic
56 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57 afwImage.Mask.getPlaneBitMask(self.config.mask))
59 image = exposureOrImage.getMaskedImage()
62 image = exposureOrImage.getImage()
64 image = exposureOrImage
66 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
70 """Configuration for combining calib images"""
71 rows = Field(doc=
"Number of rows to read at a time",
72 dtype=int, default=512)
73 mask = ListField(doc=
"Mask planes to respect", dtype=str,
74 default=[
"SAT",
"DETECTED",
"INTRP"])
75 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
76 default=int(afwMath.MEANCLIP))
77 clip = Field(doc=
"Clipping threshold for combination",
78 dtype=float, default=3.0)
79 nIter = Field(doc=
"Clipping iterations for combination",
81 stats = ConfigurableField(target=CalibStatsTask,
82 doc=
"Background statistics configuration")
86 """Task to combine calib images"""
87 ConfigClass = CalibCombineConfig
90 Task.__init__(self, *args, **kwargs)
91 self.makeSubtask(
"stats")
93 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94 """!Combine calib images for a single sensor
96 @param sensorRefList List of data references to combine (for a single sensor)
97 @param expScales List of scales to apply for each exposure
98 @param finalScale Desired scale for final combined image
99 @param inputName Data set name for inputs
100 @return combined image
103 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104 afwImage.Mask.getPlaneBitMask(self.config.mask))
107 combined = afwImage.MaskedImageF(width, height)
108 numImages = len(sensorRefList)
109 imageList = [
None]*numImages
110 for start
in range(0, height, self.config.rows):
111 rows = min(self.config.rows, height - start)
112 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113 afwGeom.Extent2I(width, rows))
114 subCombined = combined.Factory(combined, box)
116 for i, sensorRef
in enumerate(sensorRefList):
117 if sensorRef
is None:
120 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
121 if expScales
is not None:
123 imageList[i] = exposure.getMaskedImage()
125 self.
combine(subCombined, imageList, stats)
127 if finalScale
is not None:
128 background = self.stats.run(combined)
129 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
130 (NODE, background, finalScale))
131 combined *= finalScale / background
136 """Get dimensions of the inputs"""
138 for sensorRef
in sensorRefList:
139 if sensorRef
is None:
141 md = sensorRef.get(inputName +
"_md")
142 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
146 """Apply scale to input exposure
148 This implementation applies a flux scaling: the input exposure is
149 divided by the provided scale.
151 if scale
is not None:
152 mi = exposure.getMaskedImage()
156 """!Combine multiple images
158 @param target Target image to receive the combined pixels
159 @param imageList List of input images
160 @param stats Statistics control
162 images = [img
for img
in imageList
if img
is not None]
163 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
167 """Determine a consistent size, given a list of image sizes"""
168 dim = set((w, h)
for w, h
in dimList)
171 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
176 """!Return a tuple of specific values from a dict
178 This provides a hashable representation of the dict from certain keywords.
179 This can be useful for creating e.g., a tuple of the values in the DataId
180 that identify the CCD.
182 @param dict_ dict to parse
183 @param keys keys to extract (order is important)
184 @return tuple of values
186 return tuple(dict_[k]
for k
in keys)
190 """!Determine a list of CCDs from exposure references
192 This essentially inverts the exposure-level references (which
193 provides a list of CCDs for each exposure), by providing
194 a dataId list for each CCD. Consider an input list of exposures
195 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
198 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
200 This is a dict whose keys are tuples of the identifying values of a
201 CCD (usually just the CCD number) and the values are lists of dataIds
202 for that CCD in each exposure. A missing dataId is given the value
205 @param expRefList List of data references for exposures
206 @param level Level for the butler to generate CCDs
207 @param ccdKeys DataId keywords that identify a CCD
208 @return dict of data identifier lists for each CCD
210 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
211 level)]
for expRef
in expRefList]
214 ccdKeys = set(ccdKeys)
216 for ccdIdList
in expIdList:
217 for ccdId
in ccdIdList:
224 for n, ccdIdList
in enumerate(expIdList):
225 for ccdId
in ccdIdList:
227 if name
not in ccdLists:
229 ccdLists[name].append(ccdId)
235 """Split name=value pairs and put the result in a dict"""
237 def __call__(self, parser, namespace, values, option_string):
238 output = getattr(namespace, self.dest, {})
239 for nameValue
in values:
240 name, sep, valueStr = nameValue.partition(
"=")
242 parser.error(
"%s value %s must be in form name=value" %
243 (option_string, nameValue))
244 output[name] = valueStr
245 setattr(namespace, self.dest, output)
249 """ArgumentParser for calibration construction"""
252 """Add a --calibId argument to the standard pipe_base argument parser"""
253 ArgumentParser.__init__(self, *args, **kwargs)
255 self.add_id_argument(
"--id", datasetType=
"raw",
256 help=
"input identifiers, e.g., --id visit=123 ccd=4")
257 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
258 help=
"identifiers for calib, e.g., --calibId version=1",
259 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
264 Checks that the "--calibId" provided works.
266 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
268 keys = namespace.butler.getKeys(self.
calibName)
270 for name, value
in namespace.calibId.items():
273 "%s is not a relevant calib identifier key (%s)" % (name, keys))
274 parsed[name] = keys[name](value)
275 namespace.calibId = parsed
281 """Configuration for constructing calibs"""
282 clobber = Field(dtype=bool, default=
True,
283 doc=
"Clobber existing processed images?")
284 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
285 dateObs = Field(dtype=str, default=
"dateObs",
286 doc=
"Key for observation date in exposure registry")
287 dateCalib = Field(dtype=str, default=
"calibDate",
288 doc=
"Key for calib date in calib registry")
289 filter = Field(dtype=str, default=
"filter",
290 doc=
"Key for filter name in exposure/calib registries")
291 combination = ConfigurableField(
292 target=CalibCombineTask, doc=
"Calib combination configuration")
293 ccdKeys = ListField(dtype=str, default=[
294 "ccd"], doc=
"DataId keywords specifying a CCD")
295 visitKeys = ListField(dtype=str, default=[
296 "visit"], doc=
"DataId keywords specifying a visit")
297 calibKeys = ListField(dtype=str, default=[],
298 doc=
"DataId keywords specifying a calibration")
301 self.isr.doWrite =
False
305 """Get parsed values into the CalibTask.run"""
308 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
311 """Call the Task with the kwargs from getTargetList"""
312 task = self.TaskClass(config=self.config, log=self.log)
315 result = task.run(**args)
318 result = task.run(**args)
319 except Exception
as e:
322 task.log.fatal(
"Failed: %s" % e)
323 traceback.print_exc(file=sys.stderr)
325 if self.doReturnResults:
327 exitStatus=exitStatus,
329 metadata=task.metadata,
334 exitStatus=exitStatus,
338 """!Base class for constructing calibs.
340 This should be subclassed for each of the required calib types.
341 The subclass should be sure to define the following class variables:
342 * _DefaultName: default name of the task, used by CmdLineTask
343 * calibName: name of the calibration data set in the butler
344 The subclass may optionally set:
345 * filterName: filter name to give the resultant calib
347 ConfigClass = CalibConfig
348 RunnerClass = CalibTaskRunner
355 BatchPoolTask.__init__(self, *args, **kwargs)
356 self.makeSubtask(
"isr")
357 self.makeSubtask(
"combination")
361 numCcds = len(parsedCmd.butler.get(
"camera"))
362 numExps = len(cls.RunnerClass.getTargetList(
363 parsedCmd)[0][
'expRefList'])
364 numCycles = int(numCcds/float(numCores) + 0.5)
365 return time*numExps*numCycles
368 def _makeArgumentParser(cls, *args, **kwargs):
369 kwargs.pop(
"doBatch",
False)
370 return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
372 def run(self, expRefList, butler, calibId):
373 """!Construct a calib from a list of exposure references
375 This is the entry point, called by the TaskRunner.__call__
377 Only the master node executes this method.
379 @param expRefList List of data references at the exposure level
380 @param butler Data butler
381 @param calibId Identifier dict for calib
383 for expRef
in expRefList:
384 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
388 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
391 outputIdItemList = list(outputId.items())
392 for ccdName
in ccdIdLists:
393 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
394 dataId.update(outputIdItemList)
396 dataId.update(outputIdItemList)
399 butler.get(self.
calibName +
"_filename", dataId)
400 except Exception
as e:
402 "Unable to determine output filename \"%s_filename\" from %s: %s" %
406 pool.storeSet(butler=butler)
412 scales = self.
scale(ccdIdLists, data)
418 """!Generate the data identifier for the output calib
420 The mean date and the common filter are included, using keywords
421 from the configuration. The CCD-specific part is not included
422 in the data identifier.
424 @param expRefList List of data references at exposure level
425 @param calibId Data identifier elements for the calib provided by the user
426 @return data identifier
430 for expRef
in expRefList:
431 butler = expRef.getButler()
432 dataId = expRef.dataId
434 midTime += self.
getMjd(butler, dataId)
437 if filterName
is None:
438 filterName = thisFilter
439 elif filterName != thisFilter:
440 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
441 dataId, thisFilter, filterName))
443 midTime /= len(expRefList)
444 date = str(dafBase.DateTime(
445 midTime, dafBase.DateTime.MJD).toPython().date())
447 outputId = {self.config.filter: filterName,
448 self.config.dateCalib: date}
449 outputId.update(calibId)
452 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
453 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
454 if self.config.dateObs
in dataId:
455 dateObs = dataId[self.config.dateObs]
457 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
458 if "T" not in dateObs:
459 dateObs = dateObs +
"T12:00:00.0Z"
460 elif not dateObs.endswith(
"Z"):
463 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
466 """Determine the filter from a data identifier"""
467 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
471 if calibName
is None:
474 if missingKeys
is None:
475 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
477 for k
in missingKeys:
479 v = butler.queryMetadata(
'raw', [k], dataId)
480 except Exception
as e:
489 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
492 """!Update the metadata from the VisitInfo
494 \param calibImage The image whose metadata is to be set
495 \param exposureTime The exposure time for the image
496 \param darkTime The time since the last read (default: exposureTime)
500 darkTime = exposureTime
502 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
503 md = calibImage.getMetadata()
505 afwImage.setVisitInfoMetadata(md, visitInfo)
508 """!Scatter the processing among the nodes
510 We scatter each CCD independently (exposures aren't grouped together),
511 to make full use of all available processors. This necessitates piecing
512 everything back together in the same format as ccdIdLists afterwards.
514 Only the master node executes this method.
516 @param pool Process pool
517 @param ccdIdLists Dict of data identifier lists for each CCD name
518 @return Dict of lists of returned data for each CCD name
520 dataIdList = sum(ccdIdLists.values(), [])
521 self.log.info(
"Scatter processing")
523 resultList = pool.map(self.
process, dataIdList)
526 data = dict((ccdName, [
None] * len(expList))
527 for ccdName, expList
in ccdIdLists.items())
528 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
529 for expNum, dataId
in enumerate(expList)]
530 for ccdName, expList
in ccdIdLists.items()], []))
531 for dataId, result
in zip(dataIdList, resultList):
534 ccdName, expNum = indices[tuple(dataId.values())]
535 data[ccdName][expNum] = result
539 def process(self, cache, ccdId, outputName="postISRCCD"):
540 """!Process a CCD, specified by a data identifier
542 After processing, optionally returns a result (produced by
543 the 'processResult' method) calculated from the processed
544 exposure. These results will be gathered by the master node,
545 and is a means for coordinated scaling of all CCDs for flats,
548 Only slave nodes execute this method.
550 @param cache Process pool cache
551 @param ccdId Data identifier for CCD
552 @param outputName Output dataset name for butler
553 @return result from 'processResult'
556 self.log.warn(
"Null identifier received on %s" % NODE)
559 if self.config.clobber
or not sensorRef.datasetExists(outputName):
560 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
563 except Exception
as e:
564 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
570 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
571 exposure = sensorRef.get(outputName, immediate=
True)
575 """Process a single CCD, specified by a data reference
577 Generally, this simply means doing ISR.
579 Only slave nodes execute this method.
581 return self.isr.runDataRef(dataRef).exposure
584 """!Write the processed CCD
586 We need to write these out because we can't hold them all in
589 Only slave nodes execute this method.
591 @param dataRef Data reference
592 @param exposure CCD exposure to write
593 @param outputName Output dataset name for butler.
595 dataRef.put(exposure, outputName)
598 """Extract processing results from a processed exposure
600 This method generates what is gathered by the master node.
601 This can be a background measurement or similar for scaling
602 flat-fields. It must be picklable!
604 Only slave nodes execute this method.
609 """!Determine scaling across CCDs and exposures
611 This is necessary mainly for flats, so as to determine a
612 consistent scaling across the entire focal plane. This
613 implementation is simply a placeholder.
615 Only the master node executes this method.
617 @param ccdIdLists Dict of data identifier lists for each CCD tuple
618 @param data Dict of lists of returned data for each CCD tuple
619 @return dict of Struct(ccdScale: scaling for CCD,
620 expScales: scaling for each exposure
623 self.log.info(
"Scale on %s" % NODE)
624 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
625 for name
in ccdIdLists)
628 """!Scatter the combination of exposures across multiple nodes
630 In this case, we can only scatter across as many nodes as
633 Only the master node executes this method.
635 @param pool Process pool
636 @param outputId Output identifier (exposure part only)
637 @param ccdIdLists Dict of data identifier lists for each CCD name
638 @param scales Dict of structs with scales, for each CCD name
640 self.log.info(
"Scatter combination")
641 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for
642 ccdName
in ccdIdLists]
643 pool.map(self.
combine, data, outputId)
646 """!Combine multiple exposures of a particular CCD and write the output
648 Only the slave nodes execute this method.
650 @param cache Process pool cache
651 @param struct Parameters for the combination, which has the following components:
652 * ccdName Name tuple for CCD
653 * ccdIdList List of data identifiers for combination
654 * scales Scales to apply (expScales are scalings for each exposure,
655 ccdScale is final scale for combined image)
656 @param outputId Data identifier for combined image (exposure part only)
659 fullOutputId = {k: struct.ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
660 fullOutputId.update(outputId)
662 fullOutputId.update(outputId)
663 outputId = fullOutputId
666 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for
667 dataId
in struct.ccdIdList]
668 self.log.info(
"Combining %s on %s" % (outputId, NODE))
669 calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
670 finalScale=struct.scales.ccdScale)
672 if not hasattr(calib,
"getMetadata"):
673 if hasattr(calib,
"getVariance"):
674 calib = afwImage.makeExposure(calib)
676 calib = afwImage.DecoratedImageF(calib.getImage())
681 struct.ccdIdList, outputId)
685 self.
write(cache.butler, calib, outputId)
688 """!Record metadata including the inputs and creation details
690 This metadata will go into the FITS header.
692 @param butler Data butler
693 @param calib Combined calib exposure.
694 @param dataIdList List of data identifiers for calibration inputs
695 @param outputId Data identifier for output
697 header = calib.getMetadata()
701 now = time.localtime()
702 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
703 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
705 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
708 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if
710 for i, v
in enumerate(sorted(set(visits))):
711 header.add(
"CALIB_INPUT_%d" % (i,), v)
713 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
714 for key, value
in outputId.items()))
718 """Interpolate over NANs in the combined image
720 NANs can result from masked areas on the CCD. We don't want them getting
721 into our science images, so we replace them with the median of the image.
723 if hasattr(image,
"getMaskedImage"):
725 image = image.getMaskedImage().getImage()
726 if hasattr(image,
"getImage"):
727 image = image.getImage()
728 array = image.getArray()
729 bad = np.isnan(array)
730 array[bad] = np.median(array[np.logical_not(bad)])
732 def write(self, butler, exposure, dataId):
733 """!Write the final combined calib
735 Only the slave nodes execute this method
737 @param butler Data butler
738 @param exposure CCD exposure to write
739 @param dataId Data identifier for output
741 self.log.info(
"Writing %s on %s" % (dataId, NODE))
742 butler.put(exposure, self.
calibName, dataId)
746 """Configuration for bias construction.
748 No changes required compared to the base class, but
749 subclassed for distinction.
754 class BiasTask(CalibTask):
755 """Bias construction"""
756 ConfigClass = BiasConfig
757 _DefaultName =
"bias"
764 """Overrides to apply for bias construction"""
765 config.isr.doBias =
False
766 config.isr.doDark =
False
767 config.isr.doFlat =
False
768 config.isr.doFringe =
False
772 """Configuration for dark construction"""
773 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
774 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
775 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
776 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
777 repair = ConfigurableField(
778 target=RepairTask, doc=
"Task to repair artifacts")
781 CalibConfig.setDefaults(self)
782 self.combination.mask.append(
"CR")
788 The only major difference from the base class is a cosmic-ray
789 identification stage, and dividing each image by the dark time
790 to generate images of the dark rate.
792 ConfigClass = DarkConfig
793 _DefaultName =
"dark"
798 CalibTask.__init__(self, *args, **kwargs)
799 self.makeSubtask(
"repair")
803 """Overrides to apply for dark construction"""
804 config.isr.doDark =
False
805 config.isr.doFlat =
False
806 config.isr.doFringe =
False
809 """Process a single CCD
811 Besides the regular ISR, also masks cosmic-rays and divides each
812 processed image by the dark time to generate images of the dark rate.
813 The dark time is provided by the 'getDarkTime' method.
815 exposure = CalibTask.processSingle(self, sensorRef)
817 if self.config.doRepair:
818 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
819 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
821 self.repair.run(exposure, keepCRs=
False)
822 if self.config.crGrow > 0:
823 mask = exposure.getMaskedImage().getMask().clone()
824 mask &= mask.getPlaneBitMask(
"CR")
825 fpSet = afwDet.FootprintSet(
826 mask, afwDet.Threshold(0.5))
827 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
828 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
830 mi = exposure.getMaskedImage()
835 """Retrieve the dark time for an exposure"""
836 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
837 if not np.isfinite(darkTime):
838 raise RuntimeError(
"Non-finite darkTime")
843 """Configuration for flat construction"""
844 iterations = Field(dtype=int, default=10,
845 doc=
"Number of iterations for scale determination")
846 stats = ConfigurableField(target=CalibStatsTask,
847 doc=
"Background statistics configuration")
853 The principal change from the base class involves gathering the background
854 values from each image and using them to determine the scalings for the final
857 ConfigClass = FlatConfig
858 _DefaultName =
"flat"
863 """Overrides for flat construction"""
864 config.isr.doFlat =
False
865 config.isr.doFringe =
False
868 CalibTask.__init__(self, *args, **kwargs)
869 self.makeSubtask(
"stats")
872 return self.stats.run(exposure)
875 """Determine the scalings for the final combination
877 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
878 of one CCD to all the others in an exposure, and E_j is the scaling
879 of the exposure. We convert everything to logarithms so we can work
880 with a linear system. We determine the C_i and E_j from B_ij by iteration,
881 under the additional constraint that the average CCD scale is unity.
883 This algorithm comes from Eugene Magnier and Pan-STARRS.
885 assert len(ccdIdLists.values()) > 0,
"No successful CCDs"
886 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
888 lengths) == 1,
"Number of successful exposures for each CCD differs"
889 assert tuple(lengths)[0] > 0,
"No successful exposures"
891 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
892 bgMatrix = np.array([[0.0] * len(expList)
893 for expList
in ccdIdLists.values()])
894 for name
in ccdIdLists:
897 d
if d
is not None else np.nan
for d
in data[name]]
899 numpyPrint = np.get_printoptions()
900 np.set_printoptions(threshold=
'nan')
901 self.log.info(
"Input backgrounds: %s" % bgMatrix)
904 numCcds = len(ccdIdLists)
905 numExps = bgMatrix.shape[1]
907 bgMatrix = np.log(bgMatrix)
908 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
910 compScales = np.zeros(numCcds)
911 expScales = np.array(
912 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
914 for iterate
in range(self.config.iterations):
915 compScales = np.array(
916 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
917 expScales = np.array(
918 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
920 avgScale = np.average(np.exp(compScales))
921 compScales -= np.log(avgScale)
922 self.log.debug(
"Iteration %d exposure scales: %s",
923 iterate, np.exp(expScales))
924 self.log.debug(
"Iteration %d component scales: %s",
925 iterate, np.exp(compScales))
927 expScales = np.array(
928 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
930 if np.any(np.isnan(expScales)):
931 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
932 (bgMatrix, expScales))
934 expScales = np.exp(expScales)
935 compScales = np.exp(compScales)
937 self.log.info(
"Exposure scales: %s" % expScales)
938 self.log.info(
"Component relative scaling: %s" % compScales)
939 np.set_printoptions(**numpyPrint)
941 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
942 for ccdName
in ccdIdLists)
946 """Configuration for fringe construction"""
947 stats = ConfigurableField(target=CalibStatsTask,
948 doc=
"Background statistics configuration")
949 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
950 doc=
"Background configuration")
951 detection = ConfigurableField(
952 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
953 detectSigma = Field(dtype=float, default=1.0,
954 doc=
"Detection PSF gaussian sigma")
958 """Fringe construction task
960 The principal change from the base class is that the images are
961 background-subtracted and rescaled by the background.
963 XXX This is probably not right for a straight-up combination, as we
964 are currently doing, since the fringe amplitudes need not scale with
967 XXX Would like to have this do PCA and generate multiple images, but
968 that will take a bit of work with the persistence code.
970 ConfigClass = FringeConfig
971 _DefaultName =
"fringe"
976 """Overrides for fringe construction"""
977 config.isr.doFringe =
False
980 CalibTask.__init__(self, *args, **kwargs)
981 self.makeSubtask(
"detection")
982 self.makeSubtask(
"stats")
983 self.makeSubtask(
"subtractBackground")
986 """Subtract the background and normalise by the background level"""
987 exposure = CalibTask.processSingle(self, sensorRef)
988 bgLevel = self.stats.run(exposure)
989 self.subtractBackground.run(exposure)
990 mi = exposure.getMaskedImage()
992 footprintSets = self.detection.detectFootprints(
993 exposure, sigma=self.config.detectSigma)
994 mask = exposure.getMaskedImage().getMask()
995 detected = 1 << mask.addMaskPlane(
"DETECTED")
996 for fpSet
in (footprintSets.positive, footprintSets.negative):
997 if fpSet
is not None:
998 afwDet.setMaskFromFootprintList(
999 mask, fpSet.getFootprints(), detected)
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
def recordCalibInputs
Record metadata including the inputs and creation details.
def combine
Combine multiple images.
def updateMetadata
Update the metadata from the VisitInfo.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.