1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 from lsst.afw.image
import VisitInfo
20 import lsst.meas.algorithms
as measAlg
21 from lsst.pipe.tasks.repair
import RepairTask
22 from lsst.ip.isr
import IsrTask
24 from lsst.ctrl.pool.parallel
import BatchPoolTask
25 from lsst.ctrl.pool.pool
import Pool, NODE
27 from .checksum
import checksum
28 from .utils
import getDataRef
32 """Parameters controlling the measurement of background statistics"""
33 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34 default=int(afwMath.MEANCLIP))
35 clip = Field(doc=
"Clipping threshold for background",
36 dtype=float, default=3.0)
37 nIter = Field(doc=
"Clipping iterations for background",
39 mask = ListField(doc=
"Mask planes to reject",
40 dtype=str, default=[
"DETECTED",
"BAD"])
44 """Measure statistics on the background
46 This can be useful for scaling the background, e.g., for flats and fringe frames.
48 ConfigClass = CalibStatsConfig
50 def run(self, exposureOrImage):
51 """!Measure a particular statistic on an image (of some sort).
53 @param exposureOrImage Exposure, MaskedImage or Image.
54 @return Value of desired statistic
56 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57 afwImage.MaskU.getPlaneBitMask(self.config.mask))
59 image = exposureOrImage.getMaskedImage()
62 image = exposureOrImage.getImage()
64 image = exposureOrImage
66 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
70 """Configuration for combining calib images"""
71 rows = Field(doc=
"Number of rows to read at a time",
72 dtype=int, default=512)
73 mask = ListField(doc=
"Mask planes to respect", dtype=str,
74 default=[
"SAT",
"DETECTED",
"INTRP"])
75 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
76 default=int(afwMath.MEANCLIP))
77 clip = Field(doc=
"Clipping threshold for combination",
78 dtype=float, default=3.0)
79 nIter = Field(doc=
"Clipping iterations for combination",
81 stats = ConfigurableField(target=CalibStatsTask,
82 doc=
"Background statistics configuration")
86 """Task to combine calib images"""
87 ConfigClass = CalibCombineConfig
90 Task.__init__(self, *args, **kwargs)
91 self.makeSubtask(
"stats")
93 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94 """!Combine calib images for a single sensor
96 @param sensorRefList List of data references to combine (for a single sensor)
97 @param expScales List of scales to apply for each exposure
98 @param finalScale Desired scale for final combined image
99 @param inputName Data set name for inputs
100 @return combined image
103 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104 afwImage.MaskU.getPlaneBitMask(self.config.mask))
107 combined = afwImage.MaskedImageF(width, height)
108 numImages = len(sensorRefList)
109 imageList = [
None]*numImages
110 for start
in range(0, height, self.config.rows):
111 rows = min(self.config.rows, height - start)
112 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113 afwGeom.Extent2I(width, rows))
114 subCombined = combined.Factory(combined, box)
116 for i, sensorRef
in enumerate(sensorRefList):
117 if sensorRef
is None:
120 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
121 if expScales
is not None:
123 imageList[i] = exposure.getMaskedImage()
125 self.
combine(subCombined, imageList, stats)
127 if finalScale
is not None:
128 background = self.stats.run(combined)
129 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
130 (NODE, background, finalScale))
131 combined *= finalScale / background
136 """Get dimensions of the inputs"""
138 for sensorRef
in sensorRefList:
139 if sensorRef
is None:
141 md = sensorRef.get(inputName +
"_md")
142 dimList.append(afwGeom.Extent2I(
143 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
147 """Apply scale to input exposure
149 This implementation applies a flux scaling: the input exposure is
150 divided by the provided scale.
152 if scale
is not None:
153 mi = exposure.getMaskedImage()
157 """!Combine multiple images
159 @param target Target image to receive the combined pixels
160 @param imageList List of input images
161 @param stats Statistics control
163 images = [img
for img
in imageList
if img
is not None]
164 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
168 """Determine a consistent size, given a list of image sizes"""
169 dim = set((w, h)
for w, h
in dimList)
172 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
177 """!Return a tuple of specific values from a dict
179 This provides a hashable representation of the dict from certain keywords.
180 This can be useful for creating e.g., a tuple of the values in the DataId
181 that identify the CCD.
183 @param dict_ dict to parse
184 @param keys keys to extract (order is important)
185 @return tuple of values
187 return tuple(dict_[k]
for k
in keys)
191 """!Determine a list of CCDs from exposure references
193 This essentially inverts the exposure-level references (which
194 provides a list of CCDs for each exposure), by providing
195 a dataId list for each CCD. Consider an input list of exposures
196 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
199 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
201 This is a dict whose keys are tuples of the identifying values of a
202 CCD (usually just the CCD number) and the values are lists of dataIds
203 for that CCD in each exposure. A missing dataId is given the value
206 @param expRefList List of data references for exposures
207 @param level Level for the butler to generate CCDs
208 @param ccdKeys DataId keywords that identify a CCD
209 @return dict of data identifier lists for each CCD
211 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
212 level)]
for expRef
in expRefList]
215 ccdKeys = set(ccdKeys)
217 for ccdIdList
in expIdList:
218 for ccdId
in ccdIdList:
225 for n, ccdIdList
in enumerate(expIdList):
226 for ccdId
in ccdIdList:
228 if name
not in ccdLists:
230 ccdLists[name].append(ccdId)
236 """Split name=value pairs and put the result in a dict"""
238 def __call__(self, parser, namespace, values, option_string):
239 output = getattr(namespace, self.dest, {})
240 for nameValue
in values:
241 name, sep, valueStr = nameValue.partition(
"=")
243 parser.error(
"%s value %s must be in form name=value" %
244 (option_string, nameValue))
245 output[name] = valueStr
246 setattr(namespace, self.dest, output)
250 """ArgumentParser for calibration construction"""
253 """Add a --calibId argument to the standard pipe_base argument parser"""
254 ArgumentParser.__init__(self, *args, **kwargs)
256 self.add_id_argument(
"--id", datasetType=
"raw",
257 help=
"input identifiers, e.g., --id visit=123 ccd=4")
258 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
259 help=
"identifiers for calib, e.g., --calibId version=1",
260 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
265 Checks that the "--calibId" provided works.
267 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
269 keys = namespace.butler.getKeys(self.
calibName)
271 for name, value
in namespace.calibId.items():
274 "%s is not a relevant calib identifier key (%s)" % (name, keys))
275 parsed[name] = keys[name](value)
276 namespace.calibId = parsed
282 """Configuration for constructing calibs"""
283 clobber = Field(dtype=bool, default=
True,
284 doc=
"Clobber existing processed images?")
285 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
286 dateObs = Field(dtype=str, default=
"dateObs",
287 doc=
"Key for observation date in exposure registry")
288 dateCalib = Field(dtype=str, default=
"calibDate",
289 doc=
"Key for calib date in calib registry")
290 filter = Field(dtype=str, default=
"filter",
291 doc=
"Key for filter name in exposure/calib registries")
292 combination = ConfigurableField(
293 target=CalibCombineTask, doc=
"Calib combination configuration")
294 ccdKeys = ListField(dtype=str, default=[
295 "ccd"], doc=
"DataId keywords specifying a CCD")
296 visitKeys = ListField(dtype=str, default=[
297 "visit"], doc=
"DataId keywords specifying a visit")
298 calibKeys = ListField(dtype=str, default=[],
299 doc=
"DataId keywords specifying a calibration")
302 self.isr.doWrite =
False
306 """Get parsed values into the CalibTask.run"""
309 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
312 """Call the Task with the kwargs from getTargetList"""
313 task = self.TaskClass(config=self.config, log=self.log)
315 result = task.run(**args)
318 result = task.run(**args)
319 except Exception
as e:
320 task.log.fatal(
"Failed: %s" % e)
321 traceback.print_exc(file=sys.stderr)
323 if self.doReturnResults:
326 metadata=task.metadata,
332 """!Base class for constructing calibs.
334 This should be subclassed for each of the required calib types.
335 The subclass should be sure to define the following class variables:
336 * _DefaultName: default name of the task, used by CmdLineTask
337 * calibName: name of the calibration data set in the butler
338 The subclass may optionally set:
339 * filterName: filter name to give the resultant calib
341 ConfigClass = CalibConfig
342 RunnerClass = CalibTaskRunner
349 BatchPoolTask.__init__(self, *args, **kwargs)
350 self.makeSubtask(
"isr")
351 self.makeSubtask(
"combination")
355 numCcds = len(parsedCmd.butler.get(
"camera"))
356 numExps = len(cls.RunnerClass.getTargetList(
357 parsedCmd)[0][
'expRefList'])
358 numCycles = int(numCcds/float(numCores) + 0.5)
359 return time*numExps*numCycles
362 def _makeArgumentParser(cls, *args, **kwargs):
363 kwargs.pop(
"doBatch",
False)
364 return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
366 def run(self, expRefList, butler, calibId):
367 """!Construct a calib from a list of exposure references
369 This is the entry point, called by the TaskRunner.__call__
371 Only the master node executes this method.
373 @param expRefList List of data references at the exposure level
374 @param butler Data butler
375 @param calibId Identifier dict for calib
377 for expRef
in expRefList:
378 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
382 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
385 outputIdItemList = list(outputId.items())
386 for ccdName
in ccdIdLists:
387 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
389 dataId.update(outputIdItemList)
392 butler.get(self.
calibName +
"_filename", dataId)
393 except Exception
as e:
395 "Unable to determine output filename \"%s_filename\" from %s: %s" %
399 pool.storeSet(butler=butler)
405 scales = self.
scale(ccdIdLists, data)
411 """!Generate the data identifier for the output calib
413 The mean date and the common filter are included, using keywords
414 from the configuration. The CCD-specific part is not included
415 in the data identifier.
417 @param expRefList List of data references at exposure level
418 @param calibId Data identifier elements for the calib provided by the user
419 @return data identifier
423 for expRef
in expRefList:
424 butler = expRef.getButler()
425 dataId = expRef.dataId
427 midTime += self.
getMjd(butler, dataId)
430 if filterName
is None:
431 filterName = thisFilter
432 elif filterName != thisFilter:
433 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
434 dataId, thisFilter, filterName))
436 midTime /= len(expRefList)
437 date = str(dafBase.DateTime(
438 midTime, dafBase.DateTime.MJD).toPython().date())
440 outputId = {self.config.filter: filterName,
441 self.config.dateCalib: date}
442 outputId.update(calibId)
445 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
446 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
447 if self.config.dateObs
in dataId:
448 dateObs = dataId[self.config.dateObs]
450 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
451 if "T" not in dateObs:
452 dateObs = dateObs +
"T12:00:00.0Z"
453 elif not dateObs.endswith(
"Z"):
456 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
459 """Determine the filter from a data identifier"""
460 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
464 if calibName
is None:
467 if missingKeys
is None:
468 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
470 for k
in missingKeys:
472 v = butler.queryMetadata(
'raw', [k], dataId)
473 except Exception
as e:
482 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
485 """!Update the metadata from the VisitInfo
487 \param calibImage The image whose metadata is to be set
488 \param exposureTime The exposure time for the image
489 \param darkTime The time since the last read (default: exposureTime)
493 darkTime = exposureTime
495 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
496 md = calibImage.getMetadata()
499 afwImage.setVisitInfoMetadata(md, visitInfo)
500 except Exception
as e:
506 _md = dafBase.PropertyList()
509 calibImage.setMetadata(md)
511 afwImage.setVisitInfoMetadata(md, visitInfo)
514 """!Scatter the processing among the nodes
516 We scatter each CCD independently (exposures aren't grouped together),
517 to make full use of all available processors. This necessitates piecing
518 everything back together in the same format as ccdIdLists afterwards.
520 Only the master node executes this method.
522 @param pool Process pool
523 @param ccdIdLists Dict of data identifier lists for each CCD name
524 @return Dict of lists of returned data for each CCD name
526 dataIdList = sum(ccdIdLists.values(), [])
527 self.log.info(
"Scatter processing")
529 resultList = pool.map(self.
process, dataIdList)
532 data = dict((ccdName, [
None] * len(expList))
533 for ccdName, expList
in ccdIdLists.items())
534 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
535 for expNum, dataId
in enumerate(expList)]
536 for ccdName, expList
in ccdIdLists.items()], []))
537 for dataId, result
in zip(dataIdList, resultList):
540 ccdName, expNum = indices[tuple(dataId.values())]
541 data[ccdName][expNum] = result
545 def process(self, cache, ccdId, outputName="postISRCCD"):
546 """!Process a CCD, specified by a data identifier
548 After processing, optionally returns a result (produced by
549 the 'processResult' method) calculated from the processed
550 exposure. These results will be gathered by the master node,
551 and is a means for coordinated scaling of all CCDs for flats,
554 Only slave nodes execute this method.
556 @param cache Process pool cache
557 @param ccdId Data identifier for CCD
558 @param outputName Output dataset name for butler
559 @return result from 'processResult'
562 self.log.warn(
"Null identifier received on %s" % NODE)
565 if self.config.clobber
or not sensorRef.datasetExists(outputName):
566 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
569 except Exception
as e:
570 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
576 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
577 exposure = sensorRef.get(outputName, immediate=
True)
581 """Process a single CCD, specified by a data reference
583 Generally, this simply means doing ISR.
585 Only slave nodes execute this method.
587 return self.isr.runDataRef(dataRef).exposure
590 """!Write the processed CCD
592 We need to write these out because we can't hold them all in
595 Only slave nodes execute this method.
597 @param dataRef Data reference
598 @param exposure CCD exposure to write
599 @param outputName Output dataset name for butler.
601 dataRef.put(exposure, outputName)
604 """Extract processing results from a processed exposure
606 This method generates what is gathered by the master node.
607 This can be a background measurement or similar for scaling
608 flat-fields. It must be picklable!
610 Only slave nodes execute this method.
615 """!Determine scaling across CCDs and exposures
617 This is necessary mainly for flats, so as to determine a
618 consistent scaling across the entire focal plane. This
619 implementation is simply a placeholder.
621 Only the master node executes this method.
623 @param ccdIdLists Dict of data identifier lists for each CCD tuple
624 @param data Dict of lists of returned data for each CCD tuple
625 @return dict of Struct(ccdScale: scaling for CCD,
626 expScales: scaling for each exposure
629 self.log.info(
"Scale on %s" % NODE)
630 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
631 for name
in ccdIdLists)
634 """!Scatter the combination of exposures across multiple nodes
636 In this case, we can only scatter across as many nodes as
639 Only the master node executes this method.
641 @param pool Process pool
642 @param outputId Output identifier (exposure part only)
643 @param ccdIdLists Dict of data identifier lists for each CCD name
644 @param scales Dict of structs with scales, for each CCD name
646 self.log.info(
"Scatter combination")
647 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for
648 ccdName
in ccdIdLists]
649 pool.map(self.
combine, data, outputId)
652 """!Combine multiple exposures of a particular CCD and write the output
654 Only the slave nodes execute this method.
656 @param cache Process pool cache
657 @param struct Parameters for the combination, which has the following components:
658 * ccdName Name tuple for CCD
659 * ccdIdList List of data identifiers for combination
660 * scales Scales to apply (expScales are scalings for each exposure,
661 ccdScale is final scale for combined image)
662 @param outputId Data identifier for combined image (exposure part only)
665 fullOutputId = {k: struct.ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
667 fullOutputId.update(outputId)
668 outputId = fullOutputId
671 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for
672 dataId
in struct.ccdIdList]
673 self.log.info(
"Combining %s on %s" % (outputId, NODE))
674 calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
675 finalScale=struct.scales.ccdScale)
677 if not hasattr(calib,
"getMetadata"):
678 if hasattr(calib,
"getVariance"):
679 calib = afwImage.makeExposure(calib)
681 calib = afwImage.DecoratedImageF(calib.getImage())
686 struct.ccdIdList, outputId)
690 self.
write(cache.butler, calib, outputId)
693 """!Record metadata including the inputs and creation details
695 This metadata will go into the FITS header.
697 @param butler Data butler
698 @param calib Combined calib exposure.
699 @param dataIdList List of data identifiers for calibration inputs
700 @param outputId Data identifier for output
702 header = calib.getMetadata()
706 now = time.localtime()
707 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
708 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
711 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if
713 for i, v
in enumerate(sorted(set(visits))):
714 header.add(
"CALIB_INPUT_%d" % (i,), v)
716 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
717 for key, value
in outputId.items()))
721 """Interpolate over NANs in the combined image
723 NANs can result from masked areas on the CCD. We don't want them getting
724 into our science images, so we replace them with the median of the image.
726 if hasattr(image,
"getMaskedImage"):
728 image = image.getMaskedImage().getImage()
729 if hasattr(image,
"getImage"):
730 image = image.getImage()
731 array = image.getArray()
732 bad = np.isnan(array)
733 array[bad] = np.median(array[np.logical_not(bad)])
735 def write(self, butler, exposure, dataId):
736 """!Write the final combined calib
738 Only the slave nodes execute this method
740 @param butler Data butler
741 @param exposure CCD exposure to write
742 @param dataId Data identifier for output
744 self.log.info(
"Writing %s on %s" % (dataId, NODE))
745 butler.put(exposure, self.
calibName, dataId)
749 """Configuration for bias construction.
751 No changes required compared to the base class, but
752 subclassed for distinction.
757 class BiasTask(CalibTask):
758 """Bias construction"""
759 ConfigClass = BiasConfig
760 _DefaultName =
"bias"
767 """Overrides to apply for bias construction"""
768 config.isr.doBias =
False
769 config.isr.doDark =
False
770 config.isr.doFlat =
False
771 config.isr.doFringe =
False
775 """Configuration for dark construction"""
776 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
777 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
778 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
779 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
780 repair = ConfigurableField(
781 target=RepairTask, doc=
"Task to repair artifacts")
784 CalibConfig.setDefaults(self)
785 self.combination.mask.append(
"CR")
791 The only major difference from the base class is a cosmic-ray
792 identification stage, and dividing each image by the dark time
793 to generate images of the dark rate.
795 ConfigClass = DarkConfig
796 _DefaultName =
"dark"
801 CalibTask.__init__(self, *args, **kwargs)
802 self.makeSubtask(
"repair")
806 """Overrides to apply for dark construction"""
807 config.isr.doDark =
False
808 config.isr.doFlat =
False
809 config.isr.doFringe =
False
812 """Process a single CCD
814 Besides the regular ISR, also masks cosmic-rays and divides each
815 processed image by the dark time to generate images of the dark rate.
816 The dark time is provided by the 'getDarkTime' method.
818 exposure = CalibTask.processSingle(self, sensorRef)
820 if self.config.doRepair:
821 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
822 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
824 self.repair.run(exposure, keepCRs=
False)
825 if self.config.crGrow > 0:
826 mask = exposure.getMaskedImage().getMask().clone()
827 mask &= mask.getPlaneBitMask(
"CR")
828 fpSet = afwDet.FootprintSet(
829 mask, afwDet.Threshold(0.5))
830 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
831 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
833 mi = exposure.getMaskedImage()
838 """Retrieve the dark time for an exposure"""
839 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
840 if not np.isfinite(darkTime):
841 raise RuntimeError(
"Non-finite darkTime")
846 """Configuration for flat construction"""
847 iterations = Field(dtype=int, default=10,
848 doc=
"Number of iterations for scale determination")
849 stats = ConfigurableField(target=CalibStatsTask,
850 doc=
"Background statistics configuration")
856 The principal change from the base class involves gathering the background
857 values from each image and using them to determine the scalings for the final
860 ConfigClass = FlatConfig
861 _DefaultName =
"flat"
866 """Overrides for flat construction"""
867 config.isr.doFlat =
False
868 config.isr.doFringe =
False
871 CalibTask.__init__(self, *args, **kwargs)
872 self.makeSubtask(
"stats")
875 return self.stats.run(exposure)
878 """Determine the scalings for the final combination
880 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
881 of one CCD to all the others in an exposure, and E_j is the scaling
882 of the exposure. We convert everything to logarithms so we can work
883 with a linear system. We determine the C_i and E_j from B_ij by iteration,
884 under the additional constraint that the average CCD scale is unity.
886 This algorithm comes from Eugene Magnier and Pan-STARRS.
888 assert len(ccdIdLists.values()) > 0,
"No successful CCDs"
889 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
891 lengths) == 1,
"Number of successful exposures for each CCD differs"
892 assert tuple(lengths)[0] > 0,
"No successful exposures"
894 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
895 bgMatrix = np.array([[0.0] * len(expList)
896 for expList
in ccdIdLists.values()])
897 for name
in ccdIdLists:
900 d
if d
is not None else np.nan
for d
in data[name]]
902 numpyPrint = np.get_printoptions()
903 np.set_printoptions(threshold=
'nan')
904 self.log.info(
"Input backgrounds: %s" % bgMatrix)
907 numCcds = len(ccdIdLists)
908 numExps = bgMatrix.shape[1]
910 bgMatrix = np.log(bgMatrix)
911 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
913 compScales = np.zeros(numCcds)
914 expScales = np.array(
915 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
917 for iterate
in range(self.config.iterations):
918 compScales = np.array(
919 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
920 expScales = np.array(
921 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
923 avgScale = np.average(np.exp(compScales))
924 compScales -= np.log(avgScale)
925 self.log.debug(
"Iteration %d exposure scales: %s",
926 iterate, np.exp(expScales))
927 self.log.debug(
"Iteration %d component scales: %s",
928 iterate, np.exp(compScales))
930 expScales = np.array(
931 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
933 if np.any(np.isnan(expScales)):
934 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
935 (bgMatrix, expScales))
937 expScales = np.exp(expScales)
938 compScales = np.exp(compScales)
940 self.log.info(
"Exposure scales: %s" % expScales)
941 self.log.info(
"Component relative scaling: %s" % compScales)
942 np.set_printoptions(**numpyPrint)
944 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
945 for ccdName
in ccdIdLists)
949 """Configuration for fringe construction"""
950 stats = ConfigurableField(target=CalibStatsTask,
951 doc=
"Background statistics configuration")
952 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
953 doc=
"Background configuration")
954 detection = ConfigurableField(
955 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
956 detectSigma = Field(dtype=float, default=1.0,
957 doc=
"Detection PSF gaussian sigma")
961 """Fringe construction task
963 The principal change from the base class is that the images are
964 background-subtracted and rescaled by the background.
966 XXX This is probably not right for a straight-up combination, as we
967 are currently doing, since the fringe amplitudes need not scale with
970 XXX Would like to have this do PCA and generate multiple images, but
971 that will take a bit of work with the persistence code.
973 ConfigClass = FringeConfig
974 _DefaultName =
"fringe"
979 """Overrides for fringe construction"""
980 config.isr.doFringe =
False
983 CalibTask.__init__(self, *args, **kwargs)
984 self.makeSubtask(
"detection")
985 self.makeSubtask(
"stats")
986 self.makeSubtask(
"subtractBackground")
989 """Subtract the background and normalise by the background level"""
990 exposure = CalibTask.processSingle(self, sensorRef)
991 bgLevel = self.stats.run(exposure)
992 self.subtractBackground.run(exposure)
993 mi = exposure.getMaskedImage()
995 footprintSets = self.detection.detectFootprints(
996 exposure, sigma=self.config.detectSigma)
997 mask = exposure.getMaskedImage().getMask()
998 detected = 1 << mask.addMaskPlane(
"DETECTED")
999 for fpSet
in (footprintSets.positive, footprintSets.negative):
1000 if fpSet
is not None:
1001 afwDet.setMaskFromFootprintList(
1002 mask, fpSet.getFootprints(), detected)
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
def recordCalibInputs
Record metadata including the inputs and creation details.
def combine
Combine multiple images.
def updateMetadata
Update the metadata from the VisitInfo.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.