1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 import lsst.meas.algorithms
as measAlg
20 from lsst.pipe.tasks.repair
import RepairTask
21 from lsst.ip.isr
import IsrTask
23 from lsst.ctrl.pool.parallel
import BatchPoolTask
24 from lsst.ctrl.pool.pool
import Pool, NODE
26 from .checksum
import checksum
27 from .utils
import getDataRef
31 """Parameters controlling the measurement of background statistics"""
32 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
33 default=int(afwMath.MEANCLIP))
34 clip = Field(doc=
"Clipping threshold for background",
35 dtype=float, default=3.0)
36 nIter = Field(doc=
"Clipping iterations for background",
38 mask = ListField(doc=
"Mask planes to reject",
39 dtype=str, default=[
"DETECTED",
"BAD"])
43 """Measure statistics on the background
45 This can be useful for scaling the background, e.g., for flats and fringe frames.
47 ConfigClass = CalibStatsConfig
49 def run(self, exposureOrImage):
50 """!Measure a particular statistic on an image (of some sort).
52 @param exposureOrImage Exposure, MaskedImage or Image.
53 @return Value of desired statistic
55 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
56 afwImage.MaskU.getPlaneBitMask(self.config.mask))
58 image = exposureOrImage.getMaskedImage()
61 image = exposureOrImage.getImage()
63 image = exposureOrImage
65 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
69 """Configuration for combining calib images"""
70 rows = Field(doc=
"Number of rows to read at a time",
71 dtype=int, default=512)
72 mask = ListField(doc=
"Mask planes to respect", dtype=str,
73 default=[
"SAT",
"DETECTED",
"INTRP"])
74 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
75 default=int(afwMath.MEANCLIP))
76 clip = Field(doc=
"Clipping threshold for combination",
77 dtype=float, default=3.0)
78 nIter = Field(doc=
"Clipping iterations for combination",
80 stats = ConfigurableField(target=CalibStatsTask,
81 doc=
"Background statistics configuration")
85 """Task to combine calib images"""
86 ConfigClass = CalibCombineConfig
89 Task.__init__(self, *args, **kwargs)
90 self.makeSubtask(
"stats")
92 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
93 """!Combine calib images for a single sensor
95 @param sensorRefList List of data references to combine (for a single sensor)
96 @param expScales List of scales to apply for each exposure
97 @param finalScale Desired scale for final combined image
98 @param inputName Data set name for inputs
99 @return combined image
103 for mask
in self.config.mask:
104 maskVal |= afwImage.MaskU.getPlaneBitMask(mask)
105 stats = afwMath.StatisticsControl(
106 self.config.clip, self.config.nIter, maskVal)
109 combined = afwImage.MaskedImageF(width, height)
110 numImages = len(sensorRefList)
111 imageList = [
None]*numImages
112 for start
in range(0, height, self.config.rows):
113 rows = min(self.config.rows, height - start)
114 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
115 afwGeom.Extent2I(width, rows))
116 subCombined = combined.Factory(combined, box)
118 for i, sensorRef
in enumerate(sensorRefList):
119 if sensorRef
is None:
122 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
123 if expScales
is not None:
125 imageList[i] = exposure.getMaskedImage()
127 self.
combine(subCombined, imageList, stats)
129 if finalScale
is not None:
130 background = self.stats.run(combined)
131 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
132 (NODE, background, finalScale))
133 combined *= finalScale / background
135 return afwImage.DecoratedImageF(combined.getImage())
138 """Get dimensions of the inputs"""
140 for sensorRef
in sensorRefList:
141 if sensorRef
is None:
143 md = sensorRef.get(inputName +
"_md")
144 dimList.append(afwGeom.Extent2I(
145 md.get(
"NAXIS1"), md.get(
"NAXIS2")))
149 """Apply scale to input exposure
151 This implementation applies a flux scaling: the input exposure is
152 divided by the provided scale.
154 if scale
is not None:
155 mi = exposure.getMaskedImage()
159 """!Combine multiple images
161 @param target Target image to receive the combined pixels
162 @param imageList List of input images
163 @param stats Statistics control
165 images = [img
for img
in imageList
if img
is not None]
166 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
170 """Determine a consistent size, given a list of image sizes"""
171 dim = set((w, h)
for w, h
in dimList)
174 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
179 """!Return a tuple of specific values from a dict
181 This provides a hashable representation of the dict from certain keywords.
182 This can be useful for creating e.g., a tuple of the values in the DataId
183 that identify the CCD.
185 @param dict_ dict to parse
186 @param keys keys to extract (order is important)
187 @return tuple of values
189 return tuple(dict_[k]
for k
in keys)
193 """!Determine a list of CCDs from exposure references
195 This essentially inverts the exposure-level references (which
196 provides a list of CCDs for each exposure), by providing
197 a dataId list for each CCD. Consider an input list of exposures
198 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this
201 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]}
203 This is a dict whose keys are tuples of the identifying values of a
204 CCD (usually just the CCD number) and the values are lists of dataIds
205 for that CCD in each exposure. A missing dataId is given the value
208 @param expRefList List of data references for exposures
209 @param level Level for the butler to generate CCDs
210 @param ccdKeys DataId keywords that identify a CCD
211 @return dict of data identifier lists for each CCD
213 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
214 level)]
for expRef
in expRefList]
217 ccdKeys = set(ccdKeys)
219 for ccdIdList
in expIdList:
220 for ccdId
in ccdIdList:
227 for n, ccdIdList
in enumerate(expIdList):
228 for ccdId
in ccdIdList:
230 if name
not in ccdLists:
232 ccdLists[name].append(ccdId)
238 """Split name=value pairs and put the result in a dict"""
240 def __call__(self, parser, namespace, values, option_string):
241 output = getattr(namespace, self.dest, {})
242 for nameValue
in values:
243 name, sep, valueStr = nameValue.partition(
"=")
245 parser.error(
"%s value %s must be in form name=value" %
246 (option_string, nameValue))
247 output[name] = valueStr
248 setattr(namespace, self.dest, output)
252 """ArgumentParser for calibration construction"""
255 """Add a --calibId argument to the standard pipe_base argument parser"""
256 ArgumentParser.__init__(self, *args, **kwargs)
258 self.add_id_argument(
"--id", datasetType=
"raw",
259 help=
"input identifiers, e.g., --id visit=123 ccd=4")
260 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
261 help=
"identifiers for calib, e.g., --calibId version=1",
262 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
267 Checks that the "--calibId" provided works.
269 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
271 keys = namespace.butler.getKeys(self.
calibName)
273 for name, value
in namespace.calibId.items():
276 "%s is not a relevant calib identifier key (%s)" % (name, keys))
277 parsed[name] = keys[name](value)
278 namespace.calibId = parsed
284 """Configuration for constructing calibs"""
285 clobber = Field(dtype=bool, default=
True,
286 doc=
"Clobber existing processed images?")
287 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
288 dateObs = Field(dtype=str, default=
"dateObs",
289 doc=
"Key for observation date in exposure registry")
290 dateCalib = Field(dtype=str, default=
"calibDate",
291 doc=
"Key for calib date in calib registry")
292 filter = Field(dtype=str, default=
"filter",
293 doc=
"Key for filter name in exposure/calib registries")
294 combination = ConfigurableField(
295 target=CalibCombineTask, doc=
"Calib combination configuration")
296 ccdKeys = ListField(dtype=str, default=[
297 "ccd"], doc=
"DataId keywords specifying a CCD")
298 visitKeys = ListField(dtype=str, default=[
299 "visit"], doc=
"DataId keywords specifying a visit")
300 calibKeys = ListField(dtype=str, default=[],
301 doc=
"DataId keywords specifying a calibration")
304 self.isr.doWrite =
False
308 """Get parsed values into the CalibTask.run"""
311 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
314 """Call the Task with the kwargs from getTargetList"""
315 task = self.TaskClass(config=self.config, log=self.log)
317 result = task.run(**args)
320 result = task.run(**args)
321 except Exception
as e:
322 task.log.fatal(
"Failed: %s" % e)
323 traceback.print_exc(file=sys.stderr)
325 if self.doReturnResults:
328 metadata=task.metadata,
334 """!Base class for constructing calibs.
336 This should be subclassed for each of the required calib types.
337 The subclass should be sure to define the following class variables:
338 * _DefaultName: default name of the task, used by CmdLineTask
339 * calibName: name of the calibration data set in the butler
340 The subclass may optionally set:
341 * filterName: filter name to give the resultant calib
343 ConfigClass = CalibConfig
344 RunnerClass = CalibTaskRunner
350 BatchPoolTask.__init__(self, *args, **kwargs)
351 self.makeSubtask(
"isr")
352 self.makeSubtask(
"combination")
356 numCcds = len(parsedCmd.butler.get(
"camera"))
357 numExps = len(cls.RunnerClass.getTargetList(
358 parsedCmd)[0][
'expRefList'])
359 numCycles = int(numCcds/float(numCores) + 0.5)
360 return time*numExps*numCycles
363 def _makeArgumentParser(cls, *args, **kwargs):
364 kwargs.pop(
"doBatch",
False)
365 return CalibArgumentParser(calibName=cls.calibName, name=cls._DefaultName, *args, **kwargs)
367 def run(self, expRefList, butler, calibId):
368 """!Construct a calib from a list of exposure references
370 This is the entry point, called by the TaskRunner.__call__
372 Only the master node executes this method.
374 @param expRefList List of data references at the exposure level
375 @param butler Data butler
376 @param calibId Identifier dict for calib
378 for expRef
in expRefList:
379 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
383 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
386 outputIdItemList = list(outputId.items())
387 for ccdName
in ccdIdLists:
388 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
390 dataId.update(outputIdItemList)
393 butler.get(self.
calibName +
"_filename", dataId)
394 except Exception
as e:
396 "Unable to determine output filename \"%s_filename\" from %s: %s" %
400 pool.storeSet(butler=butler)
406 scales = self.
scale(ccdIdLists, data)
412 """!Generate the data identifier for the output calib
414 The mean date and the common filter are included, using keywords
415 from the configuration. The CCD-specific part is not included
416 in the data identifier.
418 @param expRefList List of data references at exposure level
419 @param calibId Data identifier elements for the calib provided by the user
420 @return data identifier
424 for expRef
in expRefList:
425 butler = expRef.getButler()
426 dataId = expRef.dataId
428 midTime += self.
getMjd(butler, dataId)
431 if filterName
is None:
432 filterName = thisFilter
433 elif filterName != thisFilter:
434 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
435 dataId, thisFilter, filterName))
437 midTime /= len(expRefList)
438 date = str(dafBase.DateTime(
439 midTime, dafBase.DateTime.MJD).toPython().date())
441 outputId = {self.config.filter: filterName,
442 self.config.dateCalib: date}
443 outputId.update(calibId)
446 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
447 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier"""
448 if self.config.dateObs
in dataId:
449 dateObs = dataId[self.config.dateObs]
451 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
452 if "T" not in dateObs:
453 dateObs = dateObs +
"T12:00:00.0Z"
454 elif not dateObs.endswith(
"Z"):
457 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
460 """Determine the filter from a data identifier"""
461 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
465 if calibName
is None:
468 if missingKeys
is None:
469 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
471 for k
in missingKeys:
473 v = butler.queryMetadata(
'raw', [k], dataId)
474 except Exception
as e:
483 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
486 """!Scatter the processing among the nodes
488 We scatter each CCD independently (exposures aren't grouped together),
489 to make full use of all available processors. This necessitates piecing
490 everything back together in the same format as ccdIdLists afterwards.
492 Only the master node executes this method.
494 @param pool Process pool
495 @param ccdIdLists Dict of data identifier lists for each CCD name
496 @return Dict of lists of returned data for each CCD name
498 dataIdList = sum(ccdIdLists.values(), [])
499 self.log.info(
"Scatter processing")
501 resultList = pool.map(self.
process, dataIdList)
504 data = dict((ccdName, [
None] * len(expList))
505 for ccdName, expList
in ccdIdLists.items())
506 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
507 for expNum, dataId
in enumerate(expList)]
508 for ccdName, expList
in ccdIdLists.items()], []))
509 for dataId, result
in zip(dataIdList, resultList):
512 ccdName, expNum = indices[tuple(dataId.values())]
513 data[ccdName][expNum] = result
517 def process(self, cache, ccdId, outputName="postISRCCD"):
518 """!Process a CCD, specified by a data identifier
520 After processing, optionally returns a result (produced by
521 the 'processResult' method) calculated from the processed
522 exposure. These results will be gathered by the master node,
523 and is a means for coordinated scaling of all CCDs for flats,
526 Only slave nodes execute this method.
528 @param cache Process pool cache
529 @param ccdId Data identifier for CCD
530 @param outputName Output dataset name for butler
531 @return result from 'processResult'
534 self.log.warn(
"Null identifier received on %s" % NODE)
537 if self.config.clobber
or not sensorRef.datasetExists(outputName):
538 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
541 except Exception
as e:
542 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
548 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
549 exposure = sensorRef.get(outputName, immediate=
True)
553 """Process a single CCD, specified by a data reference
555 Generally, this simply means doing ISR.
557 Only slave nodes execute this method.
559 return self.isr.runDataRef(dataRef).exposure
562 """!Write the processed CCD
564 We need to write these out because we can't hold them all in
567 Only slave nodes execute this method.
569 @param dataRef Data reference
570 @param exposure CCD exposure to write
571 @param outputName Output dataset name for butler.
573 dataRef.put(exposure, outputName)
576 """Extract processing results from a processed exposure
578 This method generates what is gathered by the master node.
579 This can be a background measurement or similar for scaling
580 flat-fields. It must be picklable!
582 Only slave nodes execute this method.
587 """!Determine scaling across CCDs and exposures
589 This is necessary mainly for flats, so as to determine a
590 consistent scaling across the entire focal plane. This
591 implementation is simply a placeholder.
593 Only the master node executes this method.
595 @param ccdIdLists Dict of data identifier lists for each CCD tuple
596 @param data Dict of lists of returned data for each CCD tuple
597 @return dict of Struct(ccdScale: scaling for CCD,
598 expScales: scaling for each exposure
601 self.log.info(
"Scale on %s" % NODE)
602 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
603 for name
in ccdIdLists)
606 """!Scatter the combination of exposures across multiple nodes
608 In this case, we can only scatter across as many nodes as
611 Only the master node executes this method.
613 @param pool Process pool
614 @param outputId Output identifier (exposure part only)
615 @param ccdIdLists Dict of data identifier lists for each CCD name
616 @param scales Dict of structs with scales, for each CCD name
618 self.log.info(
"Scatter combination")
619 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for
620 ccdName
in ccdIdLists]
621 pool.map(self.
combine, data, outputId)
624 """!Combine multiple exposures of a particular CCD and write the output
626 Only the slave nodes execute this method.
628 @param cache Process pool cache
629 @param struct Parameters for the combination, which has the following components:
630 * ccdName Name tuple for CCD
631 * ccdIdList List of data identifiers for combination
632 * scales Scales to apply (expScales are scalings for each exposure,
633 ccdScale is final scale for combined image)
634 @param outputId Data identifier for combined image (exposure part only)
637 fullOutputId = {k: struct.ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
639 fullOutputId.update(outputId)
640 outputId = fullOutputId
643 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for
644 dataId
in struct.ccdIdList]
645 self.log.info(
"Combining %s on %s" % (outputId, NODE))
646 calib = self.combination.run(dataRefList, expScales=struct.scales.expScales,
647 finalScale=struct.scales.ccdScale)
650 struct.ccdIdList, outputId)
654 self.
write(cache.butler, calib, outputId)
657 """!Record metadata including the inputs and creation details
659 This metadata will go into the FITS header.
661 @param butler Data butler
662 @param calib Combined calib exposure.
663 @param dataIdList List of data identifiers for calibration inputs
664 @param outputId Data identifier for output
666 header = calib.getMetadata()
670 now = time.localtime()
671 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
672 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
675 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if
677 for i, v
in enumerate(sorted(set(visits))):
678 header.add(
"CALIB_INPUT_%d" % (i,), v)
680 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
681 for key, value
in outputId.items()))
685 """Interpolate over NANs in the combined image
687 NANs can result from masked areas on the CCD. We don't want them getting
688 into our science images, so we replace them with the median of the image.
690 if hasattr(image,
"getMaskedImage"):
692 image = image.getMaskedImage().getImage()
693 if hasattr(image,
"getImage"):
694 image = image.getImage()
695 array = image.getArray()
696 bad = np.isnan(array)
697 array[bad] = np.median(array[np.logical_not(bad)])
699 def write(self, butler, exposure, dataId):
700 """!Write the final combined calib
702 Only the slave nodes execute this method
704 @param butler Data butler
705 @param exposure CCD exposure to write
706 @param dataId Data identifier for output
708 self.log.info(
"Writing %s on %s" % (dataId, NODE))
709 butler.put(exposure, self.
calibName, dataId)
713 """Configuration for bias construction.
715 No changes required compared to the base class, but
716 subclassed for distinction.
721 class BiasTask(CalibTask):
722 """Bias construction"""
723 ConfigClass = BiasConfig
724 _DefaultName =
"bias"
730 """Overrides to apply for bias construction"""
731 config.isr.doBias =
False
732 config.isr.doDark =
False
733 config.isr.doFlat =
False
734 config.isr.doFringe =
False
738 """Task to combine dark images"""
740 combined = CalibCombineTask.run(*args, **kwargs)
743 visitInfo = afwImage.VisitInfo(exposureTime=1.0, darkTime=1.0)
744 md = combined.getMetadata()
745 afwImage.setVisitInfoMetadata(md, visitInfo)
751 """Configuration for dark construction"""
752 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
753 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
754 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
755 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
756 repair = ConfigurableField(
757 target=RepairTask, doc=
"Task to repair artifacts")
760 CalibConfig.setDefaults(self)
761 self.combination.retarget(DarkCombineTask)
762 self.combination.mask.append(
"CR")
768 The only major difference from the base class is a cosmic-ray
769 identification stage, and dividing each image by the dark time
770 to generate images of the dark rate.
772 ConfigClass = DarkConfig
773 _DefaultName =
"dark"
778 CalibTask.__init__(self, *args, **kwargs)
779 self.makeSubtask(
"repair")
783 """Overrides to apply for dark construction"""
784 config.isr.doDark =
False
785 config.isr.doFlat =
False
786 config.isr.doFringe =
False
789 """Process a single CCD
791 Besides the regular ISR, also masks cosmic-rays and divides each
792 processed image by the dark time to generate images of the dark rate.
793 The dark time is provided by the 'getDarkTime' method.
795 exposure = CalibTask.processSingle(self, sensorRef)
797 if self.config.doRepair:
798 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
799 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
801 self.repair.run(exposure, keepCRs=
False)
802 if self.config.crGrow > 0:
803 mask = exposure.getMaskedImage().getMask().clone()
804 mask &= mask.getPlaneBitMask(
"CR")
805 fpSet = afwDet.FootprintSet(
806 mask, afwDet.Threshold(0.5))
807 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
808 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
810 mi = exposure.getMaskedImage()
815 """Retrieve the dark time for an exposure"""
816 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
817 if not np.isfinite(darkTime):
818 raise RuntimeError(
"Non-finite darkTime")
823 """Configuration for flat construction"""
824 iterations = Field(dtype=int, default=10,
825 doc=
"Number of iterations for scale determination")
826 stats = ConfigurableField(target=CalibStatsTask,
827 doc=
"Background statistics configuration")
833 The principal change from the base class involves gathering the background
834 values from each image and using them to determine the scalings for the final
837 ConfigClass = FlatConfig
838 _DefaultName =
"flat"
843 """Overrides for flat construction"""
844 config.isr.doFlat =
False
845 config.isr.doFringe =
False
848 CalibTask.__init__(self, *args, **kwargs)
849 self.makeSubtask(
"stats")
852 return self.stats.run(exposure)
855 """Determine the scalings for the final combination
857 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling
858 of one CCD to all the others in an exposure, and E_j is the scaling
859 of the exposure. We convert everything to logarithms so we can work
860 with a linear system. We determine the C_i and E_j from B_ij by iteration,
861 under the additional constraint that the average CCD scale is unity.
863 This algorithm comes from Eugene Magnier and Pan-STARRS.
865 assert len(ccdIdLists.values()) > 0,
"No successful CCDs"
866 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
868 lengths) == 1,
"Number of successful exposures for each CCD differs"
869 assert tuple(lengths)[0] > 0,
"No successful exposures"
871 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
872 bgMatrix = np.array([[0.0] * len(expList)
873 for expList
in ccdIdLists.values()])
874 for name
in ccdIdLists:
877 d
if d
is not None else np.nan
for d
in data[name]]
879 numpyPrint = np.get_printoptions()
880 np.set_printoptions(threshold=
'nan')
881 self.log.info(
"Input backgrounds: %s" % bgMatrix)
884 numCcds = len(ccdIdLists)
885 numExps = bgMatrix.shape[1]
887 bgMatrix = np.log(bgMatrix)
888 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
890 compScales = np.zeros(numCcds)
891 expScales = np.array(
892 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
894 for iterate
in range(self.config.iterations):
895 compScales = np.array(
896 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
897 expScales = np.array(
898 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
900 avgScale = np.average(np.exp(compScales))
901 compScales -= np.log(avgScale)
902 self.log.debug(
"Iteration %d exposure scales: %s",
903 iterate, np.exp(expScales))
904 self.log.debug(
"Iteration %d component scales: %s",
905 iterate, np.exp(compScales))
907 expScales = np.array(
908 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
910 if np.any(np.isnan(expScales)):
911 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
912 (bgMatrix, expScales))
914 expScales = np.exp(expScales)
915 compScales = np.exp(compScales)
917 self.log.info(
"Exposure scales: %s" % expScales)
918 self.log.info(
"Component relative scaling: %s" % compScales)
919 np.set_printoptions(**numpyPrint)
921 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
922 for ccdName
in ccdIdLists)
926 """Configuration for fringe construction"""
927 stats = ConfigurableField(target=CalibStatsTask,
928 doc=
"Background statistics configuration")
929 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
930 doc=
"Background configuration")
931 detection = ConfigurableField(
932 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
933 detectSigma = Field(dtype=float, default=1.0,
934 doc=
"Detection PSF gaussian sigma")
938 """Fringe construction task
940 The principal change from the base class is that the images are
941 background-subtracted and rescaled by the background.
943 XXX This is probably not right for a straight-up combination, as we
944 are currently doing, since the fringe amplitudes need not scale with
947 XXX Would like to have this do PCA and generate multiple images, but
948 that will take a bit of work with the persistence code.
950 ConfigClass = FringeConfig
951 _DefaultName =
"fringe"
956 """Overrides for fringe construction"""
957 config.isr.doFringe =
False
960 CalibTask.__init__(self, *args, **kwargs)
961 self.makeSubtask(
"detection")
962 self.makeSubtask(
"stats")
963 self.makeSubtask(
"subtractBackground")
966 """Subtract the background and normalise by the background level"""
967 exposure = CalibTask.processSingle(self, sensorRef)
968 bgLevel = self.stats.run(exposure)
969 self.subtractBackground.run(exposure)
970 mi = exposure.getMaskedImage()
972 footprintSets = self.detection.detectFootprints(
973 exposure, sigma=self.config.detectSigma)
974 mask = exposure.getMaskedImage().getMask()
975 detected = 1 << mask.addMaskPlane(
"DETECTED")
976 for fpSet
in (footprintSets.positive, footprintSets.negative):
977 if fpSet
is not None:
978 afwDet.setMaskFromFootprintList(
979 mask, fpSet.getFootprints(), detected)
def run
Measure a particular statistic on an image (of some sort).
def getCcdIdListFromExposures
Determine a list of CCDs from exposure references.
def scatterProcess
Scatter the processing among the nodes.
def checksum
Calculate a checksum of an object.
def recordCalibInputs
Record metadata including the inputs and creation details.
def combine
Combine multiple images.
def getOutputId
Generate the data identifier for the output calib.
def write
Write the final combined calib.
def run
Construct a calib from a list of exposure references.
def combine
Combine multiple exposures of a particular CCD and write the output.
def scale
Determine scaling across CCDs and exposures.
def scatterCombine
Scatter the combination of exposures across multiple nodes.
def run
Combine calib images for a single sensor.
def process
Process a CCD, specified by a data identifier.
def dictToTuple
Return a tuple of specific values from a dict.
def processWrite
Write the processed CCD.
Base class for constructing calibs.