1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 import lsst.meas.algorithms
as measAlg
20 from lsst.pipe.tasks.repair
import RepairTask
21 from lsst.ip.isr
import IsrTask
23 from lsst.ctrl.pool.parallel
import BatchPoolTask
24 from lsst.ctrl.pool.pool
import Pool, NODE
26 from .checksum
import checksum
27 from .utils
import getDataRef
31 """Parameters controlling the measurement of background statistics"""
32 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
33 default=int(afwMath.MEANCLIP))
34 clip = Field(doc=
"Clipping threshold for background",
35 dtype=float, default=3.0)
36 nIter = Field(doc=
"Clipping iterations for background",
38 mask = ListField(doc=
"Mask planes to reject",
39 dtype=str, default=[
"DETECTED",
"BAD"])
43 """Measure statistics on the background
45 This can be useful for scaling the background, e.g., for flats and fringe frames.
47 ConfigClass = CalibStatsConfig
49 def run(self, exposureOrImage):
50 """!Measure a particular statistic on an image (of some sort).
52 @param exposureOrImage Exposure, MaskedImage or Image.
53 @return Value of desired statistic
55 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
56 afwImage.MaskU.getPlaneBitMask(self.config.mask))
58 image = exposureOrImage.getMaskedImage()
61 image = exposureOrImage.getImage()
63 image = exposureOrImage
65 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
69 """Configuration for combining calib images"""
70 rows = Field(doc=
"Number of rows to read at a time",
71 dtype=int, default=512)
72 mask = ListField(doc=
"Mask planes to respect", dtype=str,
73 default=[
"SAT",
"DETECTED",
"INTRP"])
74 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
75 default=int(afwMath.MEANCLIP))
76 clip = Field(doc=
"Clipping threshold for combination",
77 dtype=float, default=3.0)
78 nIter = Field(doc=
"Clipping iterations for combination",
80 stats = ConfigurableField(target=CalibStatsTask,
81 doc=
"Background statistics configuration")
85 """Task to combine calib images"""
86 ConfigClass = CalibCombineConfig
89 Task.__init__(self, *args, **kwargs)
90 self.makeSubtask(
"stats")
92 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
93 """!Combine calib images for a single sensor
95 @param sensorRefList List of data references to combine (for a single sensor)
96 @param expScales List of scales to apply for each exposure
97 @param finalScale Desired scale for final combined image
98 @param inputName Data set name for inputs
99 @return combined image
103 for mask
in self.config.mask:
104 maskVal |= afwImage.MaskU.getPlaneBitMask(mask)
105 stats = afwMath.StatisticsControl(
106 self.config.clip, self.config.nIter, maskVal)
109 combined = afwImage.MaskedImageF(width, height)
110 numImages = len(sensorRefList)
111 imageList = [
None]*numImages
112 for start
in range(0, height, self.config.rows):
113 rows = min(self.config.rows, height - start)
114 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
115 afwGeom.Extent2I(width, rows))
116 subCombined = combined.Factory(combined, box)
118 for i, sensorRef
in enumerate(sensorRefList):
119 if sensorRef
is None:
122 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
123 if expScales
is not None:
125 imageList[i] = exposure.getMaskedImage()
127 self.
combine(subCombined, imageList, stats)
129 if finalScale
is not None:
130 background = self.stats.run(combined)
131 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
132 (NODE, background, finalScale))
133 combined *= finalScale / background
135 return afwImage.DecoratedImageF(combined.getImage())
138 """Get dimensions of the inputs"""
140 for sensorRef
in sensorRefList:
141 if sensorRef
is None:
143 md = sensorRef.get(inputName +
"_md")
144 dimList.append(afwGeom.Extent2I(
145 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
149 """Apply scale to input exposure
151 This implementation applies a flux scaling: the input exposure is
152 divided by the provided scale.
154 if scale
is not None:
155 mi = exposure.getMaskedImage()
159 """!Combine multiple images
161 @param target Target image to receive the combined pixels
162 @param imageList List of input images
163 @param stats Statistics control
165 images = [img
for img
in imageList
if img
is not None]
166 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
170 """Determine a consistent size, given a list of image sizes"""
171 dim = set((w, h)
for w, h
in dimList)
174 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
179 """!Return a tuple of specific values from a dict
181 This provides a hashable representation of the dict from certain keywords.
182 This can be useful for creating e.g., a tuple of the values in the DataId
183 that identify the CCD.
185 @param dict_ dict to parse
186 @param keys keys to extract (order is important)
187 @return tuple of values
189 return tuple(dict_[k]
for k
in keys)
193 """!Determine a list of CCDs from exposure references
195 This essentially inverts the exposure-level references (which
196 provides a list of CCDs for each exposure), by providing
197 a dataId list for each CCD. Consider an input list of exposures
198 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
201 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
203 This is a dict whose keys are tuples of the identifying values of a
204 CCD (usually just the CCD number) and the values are lists of dataIds
205 for that CCD in each exposure. A missing dataId is given the value
208 @param expRefList List of data references for exposures
209 @param level Level for the butler to generate CCDs
210 @param ccdKeys DataId keywords that identify a CCD
211 @return dict of data identifier lists for each CCD
213 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
214 level)]
for expRef
in expRefList]
217 ccdKeys = set(ccdKeys)
219 for ccdIdList
in expIdList:
220 for ccdId
in ccdIdList:
227 for n, ccdIdList
in enumerate(expIdList):
228 for ccdId
in ccdIdList:
230 if name
not in ccdLists:
232 ccdLists[name].append(ccdId)
238 """Split name=value pairs and put the result in a dict"""
240 def __call__(self, parser, namespace, values, option_string):
241 output = getattr(namespace, self.dest, {})
242 for nameValue
in values:
243 name, sep, valueStr = nameValue.partition(
"=")
245 parser.error(
"%s value %s must be in form name=value" %
246 (option_string, nameValue))
247 output[name] = valueStr
248 setattr(namespace, self.dest, output)
252 """ArgumentParser for calibration construction"""
255 """Add a --calibId argument to the standard pipe_base argument parser"""
256 ArgumentParser.__init__(self, *args, **kwargs)
258 self.add_id_argument(
"--id", datasetType=
"raw",
259 help=
"input identifiers, e.g., --id visit=123 ccd=4")
260 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
261 help=
"identifiers for calib, e.g., --calibId version=1",
262 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
267 Checks that the "--calibId" provided works.
269 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
271 keys = namespace.butler.getKeys(self.
calibName)
273 for name, value
in namespace.calibId.items():
276 "%s is not a relevant calib identifier key (%s)" % (name, keys))
277 parsed[name] = keys[name](value)
278 namespace.calibId = parsed
284 """Configuration for constructing calibs"""
285 clobber = Field(dtype=bool, default=
True,
286 doc=
"Clobber existing processed images?")
287 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
288 dateObs = Field(dtype=str, default=
"dateObs",
289 doc=
"Key for observation date in exposure registry")
290 dateCalib = Field(dtype=str, default=
"calibDate",
291 doc=
"Key for calib date in calib registry")
292 filter = Field(dtype=str, default=
"filter",
293 doc=
"Key for filter name in exposure/calib registries")
294 combination = ConfigurableField(
295 target=CalibCombineTask, doc=
"Calib combination configuration")
296 ccdKeys = ListField(dtype=str, default=[
297 "ccd"], doc=
"DataId keywords specifying a CCD")
298 visitKeys = ListField(dtype=str, default=[
299 "visit"], doc=
"DataId keywords specifying a visit")
300 calibKeys = ListField(dtype=str, default=[],
301 doc=
"DataId keywords specifying a calibration")
304 self.isr.doWrite =
False
308 """Get parsed values into the CalibTask.run"""
311 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
314 """Call the Task with the kwargs from getTargetList"""
315 task = self.TaskClass(config=self.config, log=self.log)
317 result = task.run(**args)
320 result = task.run(**args)
321 except Exception
as e:
322 task.log.fatal(
"Failed: %s" % e)
323 traceback.print_exc(file=sys.stderr)
325 if self.doReturnResults:
328 metadata=task.metadata,
334 """!Base class for constructing calibs.
336 This should be subclassed for each of the required calib types.
337 The subclass should be sure to define the following class variables:
338 * _DefaultName: default name of the task, used by CmdLineTask
339 * calibName: name of the calibration data set in the butler
340 The subclass may optionally set:
341 * filterName: filter name to give the resultant calib
343 ConfigClass = CalibConfig
344 RunnerClass = CalibTaskRunner
350 BatchPoolTask.__init__(self, *args, **kwargs)
351 self.makeSubtask(
"isr")
352 self.makeSubtask(
"combination")
356 numCcds = len(parsedCmd.butler.get(
"camera"))
357 numExps = len(cls.RunnerClass.getTargetList(
358 parsedCmd)[0][
'expRefList'])
359 numCycles = int(numCcds/float(numCores) + 0.5)
360 return time*numExps*numCycles
363 def _makeArgumentParser(cls, *args, **kwargs):
364 kwargs.pop(
"doBatch",
False)
365 return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
367 def run(self, expRefList, butler, calibId):
368 """!Construct a calib from a list of exposure references
370 This is the entry point, called by the TaskRunner.__call__
372 Only the master node executes this method.
374 @param expRefList List of data references at the exposure level
375 @param butler Data butler
376 @param calibId Identifier dict for calib
380 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
383 outputIdItemList = list(outputId.items())
384 for ccdName
in ccdIdLists:
385 dataId = dict(outputIdItemList + [(k, ccdName[i])
386 for i, k
in enumerate(self.config.ccdKeys)])
388 butler.get(self.
calibName +
"_filename", dataId)
389 except Exception
as e:
391 "Unable to determine output filename from %s: %s" % (dataId, e))
394 pool.storeSet(butler=butler)
400 scales = self.
scale(ccdIdLists, data)
406 """!Generate the data identifier for the output calib
408 The mean date and the common filter are included, using keywords
409 from the configuration. The CCD-specific part is not included
410 in the data identifier.
412 @param expRefList List of data references at exposure level
413 @param calibId Data identifier elements for the calib provided by the user
414 @return data identifier
418 for expRef
in expRefList:
419 butler = expRef.getButler()
420 dataId = expRef.dataId
422 midTime += self.
getMjd(butler, dataId)
425 if filterName
is None:
426 filterName = thisFilter
427 elif filterName != thisFilter:
428 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
429 dataId, thisFilter, filterName))
431 midTime /= len(expRefList)
432 date = str(dafBase.DateTime(
433 midTime, dafBase.DateTime.MJD).toPython().date())
435 outputId = {self.config.filter: filterName,
436 self.config.dateCalib: date}
437 outputId.update(calibId)
440 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
441 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
442 if self.config.dateObs
in dataId:
443 dateObs = dataId[self.config.dateObs]
445 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
446 if "T" not in dateObs:
447 dateObs = dateObs +
"T12:00:00.0Z"
448 elif not dateObs.endswith(
"Z"):
451 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
454 """Determine the filter from a data identifier"""
455 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
459 """!Scatter the processing among the nodes
461 We scatter each CCD independently (exposures aren't grouped together),
462 to make full use of all available processors. This necessitates piecing
463 everything back together in the same format as ccdIdLists afterwards.
465 Only the master node executes this method.
467 @param pool Process pool
468 @param ccdIdLists Dict of data identifier lists for each CCD name
469 @return Dict of lists of returned data for each CCD name
471 dataIdList = sum(ccdIdLists.values(), [])
472 self.log.info(
"Scatter processing")
474 resultList = pool.map(self.
process, dataIdList)
477 data = dict((ccdName, [
None] * len(expList))
478 for ccdName, expList
in ccdIdLists.items())
479 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
480 for expNum, dataId
in enumerate(expList)]
481 for ccdName, expList
in ccdIdLists.items()], []))
482 for dataId, result
in zip(dataIdList, resultList):
485 ccdName, expNum = indices[tuple(dataId.values())]
486 data[ccdName][expNum] = result
490 def process(self, cache, ccdId, outputName="postISRCCD"):
491 """!Process a CCD, specified by a data identifier
493 After processing, optionally returns a result (produced by
494 the 'processResult' method) calculated from the processed
495 exposure. These results will be gathered by the master node,
496 and is a means for coordinated scaling of all CCDs for flats,
499 Only slave nodes execute this method.
501 @param cache Process pool cache
502 @param ccdId Data identifier for CCD
503 @param outputName Output dataset name for butler
504 @return result from 'processResult'
507 self.log.warn(
"Null identifier received on %s" % NODE)
510 if self.config.clobber
or not sensorRef.datasetExists(outputName):
511 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
514 except Exception
as e:
515 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
521 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
522 exposure = sensorRef.get(outputName, immediate=
True)
526 """Process a single CCD, specified by a data reference
528 Generally, this simply means doing ISR.
530 Only slave nodes execute this method.
532 return self.isr.runDataRef(dataRef).exposure
535 """!Write the processed CCD
537 We need to write these out because we can't hold them all in
540 Only slave nodes execute this method.
542 @param dataRef Data reference
543 @param exposure CCD exposure to write
544 @param outputName Output dataset name for butler.
546 dataRef.put(exposure, outputName)
549 """Extract processing results from a processed exposure
551 This method generates what is gathered by the master node.
552 This can be a background measurement or similar for scaling
553 flat-fields. It must be picklable!
555 Only slave nodes execute this method.
560 """!Determine scaling across CCDs and exposures
562 This is necessary mainly for flats, so as to determine a
563 consistent scaling across the entire focal plane. This
564 implementation is simply a placeholder.
566 Only the master node executes this method.
568 @param ccdIdLists Dict of data identifier lists for each CCD tuple
569 @param data Dict of lists of returned data for each CCD tuple
570 @return dict of Struct(ccdScale: scaling for CCD,
571 expScales: scaling for each exposure
574 self.log.info(
"Scale on %s" % NODE)
575 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
576 for name
in ccdIdLists)
579 """!Scatter the combination of exposures across multiple nodes
581 In this case, we can only scatter across as many nodes as
584 Only the master node executes this method.
586 @param pool Process pool
587 @param outputId Output identifier (exposure part only)
588 @param ccdIdLists Dict of data identifier lists for each CCD name
589 @param scales Dict of structs with scales, for each CCD name
591 self.log.info(
"Scatter combination")
592 outputIdItemList = outputId.items()
593 data = [Struct(ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName],
594 outputId=dict(outputIdItemList +
595 [(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)]))
for
596 ccdName
in ccdIdLists]
600 """!Combine multiple exposures of a particular CCD and write the output
602 Only the slave nodes execute this method.
604 @param cache Process pool cache
605 @param struct Parameters for the combination, which has the following components:
606 * ccdIdList List of data identifiers for combination
607 * scales Scales to apply (expScales are scalings for each exposure,
608 ccdScale is final scale for combined image)
609 * outputId Data identifier for combined image (fully qualified for this CCD)
611 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for
612 dataId
in struct.ccdIdList]
613 self.log.info(
"Combining %s on %s" % (struct.outputId, NODE))
614 calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
615 finalScale=struct.scales.ccdScale)
618 struct.ccdIdList, struct.outputId)
622 self.
write(cache.butler, calib, struct.outputId)
625 """!Record metadata including the inputs and creation details
627 This metadata will go into the FITS header.
629 @param butler Data butler
630 @param calib Combined calib exposure.
631 @param dataIdList List of data identifiers for calibration inputs
632 @param outputId Data identifier for output
634 header = calib.getMetadata()
638 now = time.localtime()
639 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
640 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
643 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if
645 for i, v
in enumerate(sorted(set(visits))):
646 header.add(
"CALIB_INPUT_%d" % (i,), v)
648 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
649 for key, value
in outputId.items()))
653 """Interpolate over NANs in the combined image
655 NANs can result from masked areas on the CCD. We don't want them getting
656 into our science images, so we replace them with the median of the image.
658 if hasattr(image,
"getMaskedImage"):
660 image = image.getMaskedImage().getImage()
661 if hasattr(image,
"getImage"):
662 image = image.getImage()
663 array = image.getArray()
664 bad = np.isnan(array)
665 array[bad] = np.median(array[np.logical_not(bad)])
667 def write(self, butler, exposure, dataId):
668 """!Write the final combined calib
670 Only the slave nodes execute this method
672 @param butler Data butler
673 @param exposure CCD exposure to write
674 @param dataId Data identifier for output
676 self.log.info(
"Writing %s on %s" % (dataId, NODE))
677 butler.put(exposure, self.
calibName, dataId)
681 """Configuration for bias construction.
683 No changes required compared to the base class, but
684 subclassed for distinction.
689 class BiasTask(CalibTask):
690 """Bias construction"""
691 ConfigClass = BiasConfig
692 _DefaultName =
"bias"
698 """Overrides to apply for bias construction"""
699 config.isr.doBias =
False
700 config.isr.doDark =
False
701 config.isr.doFlat =
False
702 config.isr.doFringe =
False
706 """Task to combine dark images"""
708 combined = CalibCombineTask.run(*args, **kwargs)
711 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
712 md = combined.getMetadata()
713 afwImage.setVisitInfoMetadata(md, visitInfo)
719 """Configuration for dark construction"""
720 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
721 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
722 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
723 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
724 repair = ConfigurableField(
725 target=RepairTask, doc=
"Task to repair artifacts")
728 CalibConfig.setDefaults(self)
729 self.combination.retarget(DarkCombineTask)
730 self.combination.mask.append(
"CR")
736 The only major difference from the base class is a cosmic-ray
737 identification stage, and dividing each image by the dark time
738 to generate images of the dark rate.
740 ConfigClass = DarkConfig
741 _DefaultName =
"dark"
746 CalibTask.__init__(self, *args, **kwargs)
747 self.makeSubtask(
"repair")
751 """Overrides to apply for dark construction"""
752 config.isr.doDark =
False
753 config.isr.doFlat =
False
754 config.isr.doFringe =
False
757 """Process a single CCD
759 Besides the regular ISR, also masks cosmic-rays and divides each
760 processed image by the dark time to generate images of the dark rate.
761 The dark time is provided by the 'getDarkTime' method.
763 exposure = CalibTask.processSingle(self, sensorRef)
765 if self.config.doRepair:
766 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
767 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
769 self.repair.run(exposure, keepCRs=
False)
770 if self.config.crGrow > 0:
771 mask = exposure.getMaskedImage().getMask().clone()
772 mask &= mask.getPlaneBitMask(
"CR")
773 fpSet = afwDet.FootprintSet(
774 mask.convertU(), afwDet.Threshold(0.5))
775 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
776 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
778 mi = exposure.getMaskedImage()
783 """Retrieve the dark time for an exposure"""
784 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
785 if not np.isfinite(darkTime):
786 raise RuntimeError(
"Non-finite darkTime")
791 """Configuration for flat construction"""
792 iterations = Field(dtype=int, default=10,
793 doc=
"Number of iterations for scale determination")
794 stats = ConfigurableField(target=CalibStatsTask,
795 doc=
"Background statistics configuration")
801 The principal change from the base class involves gathering the background
802 values from each image and using them to determine the scalings for the final
805 ConfigClass = FlatConfig
806 _DefaultName =
"flat"
811 """Overrides for flat construction"""
812 config.isr.doFlat =
False
813 config.isr.doFringe =
False
816 CalibTask.__init__(self, *args, **kwargs)
817 self.makeSubtask(
"stats")
820 return self.stats.run(exposure)
823 """Determine the scalings for the final combination
825 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
826 of one CCD to all the others in an exposure, and E_j is the scaling
827 of the exposure. We convert everything to logarithms so we can work
828 with a linear system. We determine the C_i and E_j from B_ij by iteration,
829 under the additional constraint that the average CCD scale is unity.
831 This algorithm comes from Eugene Magnier and Pan-STARRS.
833 assert len(ccdIdLists.values()) > 0,
"No successful CCDs"
834 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
836 lengths) == 1,
"Number of successful exposures for each CCD differs"
837 assert tuple(lengths)[0] > 0,
"No successful exposures"
839 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
840 bgMatrix = np.array([[0.0] * len(expList)
841 for expList
in ccdIdLists.values()])
842 for name
in ccdIdLists:
845 d
if d
is not None else np.nan
for d
in data[name]]
847 numpyPrint = np.get_printoptions()
848 np.set_printoptions(threshold=
'nan')
849 self.log.info(
"Input backgrounds: %s" % bgMatrix)
852 numCcds = len(ccdIdLists)
853 numExps = bgMatrix.shape[1]
855 bgMatrix = np.log(bgMatrix)
856 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
858 compScales = np.zeros(numCcds)
859 expScales = np.array(
860 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
862 for iterate
in range(self.config.iterations):
863 compScales = np.array(
864 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
865 expScales = np.array(
866 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
868 avgScale = np.average(np.exp(compScales))
869 compScales -= np.log(avgScale)
870 self.log.debug(
"Iteration %d exposure scales: %s",
871 iterate, np.exp(expScales))
872 self.log.debug(
"Iteration %d component scales: %s",
873 iterate, np.exp(compScales))
875 expScales = np.array(
876 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
878 if np.any(np.isnan(expScales)):
879 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
880 (bgMatrix, expScales))
882 expScales = np.exp(expScales)
883 compScales = np.exp(compScales)
885 self.log.info(
"Exposure scales: %s" % expScales)
886 self.log.info(
"Component relative scaling: %s" % compScales)
887 np.set_printoptions(**numpyPrint)
889 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
890 for ccdName
in ccdIdLists)
894 """Configuration for fringe construction"""
895 stats = ConfigurableField(target=CalibStatsTask,
896 doc=
"Background statistics configuration")
897 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
898 doc=
"Background configuration")
899 detection = ConfigurableField(
900 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
901 detectSigma = Field(dtype=float, default=1.0,
902 doc=
"Detection PSF gaussian sigma")
906 """Fringe construction task
908 The principal change from the base class is that the images are
909 background-subtracted and rescaled by the background.
911 XXX This is probably not right for a straight-up combination, as we
912 are currently doing, since the fringe amplitudes need not scale with
915 XXX Would like to have this do PCA and generate multiple images, but
916 that will take a bit of work with the persistence code.
918 ConfigClass = FringeConfig
919 _DefaultName =
"fringe"
924 """Overrides for fringe construction"""
925 config.isr.doFringe =
False
928 CalibTask.__init__(self, *args, **kwargs)
929 self.makeSubtask(
"detection")
930 self.makeSubtask(
"stats")
931 self.makeSubtask(
"subtractBackground")
934 """Subtract the background and normalise by the background level"""
935 exposure = CalibTask.processSingle(self, sensorRef)
936 bgLevel = self.stats.run(exposure)
937 self.subtractBackground.run(exposure)
938 mi = exposure.getMaskedImage()
940 footprintSets = self.detection.detectFootprints(
941 exposure, sigma=self.config.detectSigma)
942 mask = exposure.getMaskedImage().getMask()
943 detected = 1 << mask.addMaskPlane(
"DETECTED")
944 for fpSet
in (footprintSets.positive, footprintSets.negative):
945 if fpSet
is not None:
946 afwDet.setMaskFromFootprintList(
947 mask, fpSet.getFootprints(), detected)
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
def recordCalibInputs
Record metadata including the inputs and creation details.
def combine
Combine multiple images.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.