1 from __future__
import absolute_import, division, print_function
9 from builtins
import zip
10 from builtins
import range
12 from lsst.pex.config
import Config, ConfigurableField, Field, ListField
13 from lsst.pipe.base
import Task, Struct, TaskRunner, ArgumentParser
14 import lsst.daf.base
as dafBase
15 import lsst.afw.math
as afwMath
16 import lsst.afw.geom
as afwGeom
17 import lsst.afw.detection
as afwDet
18 import lsst.afw.image
as afwImage
19 from lsst.afw.image
import VisitInfo
20 import lsst.meas.algorithms
as measAlg
21 from lsst.pipe.tasks.repair
import RepairTask
22 from lsst.ip.isr
import IsrTask
24 from lsst.ctrl.pool.parallel
import BatchPoolTask
25 from lsst.ctrl.pool.pool
import Pool, NODE
27 from .checksum
import checksum
28 from .utils
import getDataRef
32 """Parameters controlling the measurement of background statistics""" 33 stat = Field(doc=
"Statistic to use to estimate background (from lsst.afw.math)", dtype=int,
34 default=int(afwMath.MEANCLIP))
35 clip = Field(doc=
"Clipping threshold for background",
36 dtype=float, default=3.0)
37 nIter = Field(doc=
"Clipping iterations for background",
39 mask = ListField(doc=
"Mask planes to reject",
40 dtype=str, default=[
"DETECTED",
"BAD"])
44 """Measure statistics on the background 46 This can be useful for scaling the background, e.g., for flats and fringe frames. 48 ConfigClass = CalibStatsConfig
50 def run(self, exposureOrImage):
51 """!Measure a particular statistic on an image (of some sort). 53 @param exposureOrImage Exposure, MaskedImage or Image. 54 @return Value of desired statistic 56 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
57 afwImage.Mask.getPlaneBitMask(self.config.mask))
59 image = exposureOrImage.getMaskedImage()
62 image = exposureOrImage.getImage()
64 image = exposureOrImage
66 return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
70 """Configuration for combining calib images""" 71 rows = Field(doc=
"Number of rows to read at a time",
72 dtype=int, default=512)
73 mask = ListField(doc=
"Mask planes to respect", dtype=str,
74 default=[
"SAT",
"DETECTED",
"INTRP"])
75 combine = Field(doc=
"Statistic to use for combination (from lsst.afw.math)", dtype=int,
76 default=int(afwMath.MEANCLIP))
77 clip = Field(doc=
"Clipping threshold for combination",
78 dtype=float, default=3.0)
79 nIter = Field(doc=
"Clipping iterations for combination",
81 stats = ConfigurableField(target=CalibStatsTask,
82 doc=
"Background statistics configuration")
86 """Task to combine calib images""" 87 ConfigClass = CalibCombineConfig
90 Task.__init__(self, *args, **kwargs)
91 self.makeSubtask(
"stats")
93 def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD"):
94 """!Combine calib images for a single sensor 96 @param sensorRefList List of data references to combine (for a single sensor) 97 @param expScales List of scales to apply for each exposure 98 @param finalScale Desired scale for final combined image 99 @param inputName Data set name for inputs 100 @return combined image 103 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
104 afwImage.Mask.getPlaneBitMask(self.config.mask))
107 combined = afwImage.MaskedImageF(width, height)
108 numImages = len(sensorRefList)
109 imageList = [
None]*numImages
110 for start
in range(0, height, self.config.rows):
111 rows = min(self.config.rows, height - start)
112 box = afwGeom.Box2I(afwGeom.Point2I(0, start),
113 afwGeom.Extent2I(width, rows))
114 subCombined = combined.Factory(combined, box)
116 for i, sensorRef
in enumerate(sensorRefList):
117 if sensorRef
is None:
120 exposure = sensorRef.get(inputName +
"_sub", bbox=box)
121 if expScales
is not None:
123 imageList[i] = exposure.getMaskedImage()
125 self.
combine(subCombined, imageList, stats)
127 if finalScale
is not None:
128 background = self.stats.
run(combined)
129 self.log.info(
"%s: Measured background of stack is %f; adjusting to %f" %
130 (NODE, background, finalScale))
131 combined *= finalScale / background
136 """Get dimensions of the inputs""" 138 for sensorRef
in sensorRefList:
139 if sensorRef
is None:
141 md = sensorRef.get(inputName +
"_md")
142 dimList.append(afwImage.bboxFromMetadata(md).
getDimensions())
146 """Apply scale to input exposure 148 This implementation applies a flux scaling: the input exposure is 149 divided by the provided scale. 151 if scale
is not None:
152 mi = exposure.getMaskedImage()
156 """!Combine multiple images 158 @param target Target image to receive the combined pixels 159 @param imageList List of input images 160 @param stats Statistics control 162 images = [img
for img
in imageList
if img
is not None]
163 afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
167 """Determine a consistent size, given a list of image sizes""" 168 dim = set((w, h)
for w, h
in dimList)
171 raise RuntimeError(
"Inconsistent dimensions: %s" % dim)
176 """!Return a tuple of specific values from a dict 178 This provides a hashable representation of the dict from certain keywords. 179 This can be useful for creating e.g., a tuple of the values in the DataId 180 that identify the CCD. 182 @param dict_ dict to parse 183 @param keys keys to extract (order is important) 184 @return tuple of values 186 return tuple(dict_[k]
for k
in keys)
190 """!Determine a list of CCDs from exposure references 192 This essentially inverts the exposure-level references (which 193 provides a list of CCDs for each exposure), by providing 194 a dataId list for each CCD. Consider an input list of exposures 195 [e1, e2, e3], and each exposure has CCDs c1 and c2. Then this 198 {(c1,): [e1c1, e2c1, e3c1], (c2,): [e1c2, e2c2, e3c2]} 200 This is a dict whose keys are tuples of the identifying values of a 201 CCD (usually just the CCD number) and the values are lists of dataIds 202 for that CCD in each exposure. A missing dataId is given the value 205 @param expRefList List of data references for exposures 206 @param level Level for the butler to generate CCDs 207 @param ccdKeys DataId keywords that identify a CCD 208 @return dict of data identifier lists for each CCD 210 expIdList = [[ccdRef.dataId
for ccdRef
in expRef.subItems(
211 level)]
for expRef
in expRefList]
214 ccdKeys = set(ccdKeys)
216 for ccdIdList
in expIdList:
217 for ccdId
in ccdIdList:
224 for n, ccdIdList
in enumerate(expIdList):
225 for ccdId
in ccdIdList:
227 if name
not in ccdLists:
229 ccdLists[name].append(ccdId)
235 """Split name=value pairs and put the result in a dict""" 237 def __call__(self, parser, namespace, values, option_string):
238 output = getattr(namespace, self.dest, {})
239 for nameValue
in values:
240 name, sep, valueStr = nameValue.partition(
"=")
242 parser.error(
"%s value %s must be in form name=value" %
243 (option_string, nameValue))
244 output[name] = valueStr
245 setattr(namespace, self.dest, output)
249 """ArgumentParser for calibration construction""" 252 """Add a --calibId argument to the standard pipe_base argument parser""" 253 ArgumentParser.__init__(self, *args, **kwargs)
255 self.add_id_argument(
"--id", datasetType=
"raw",
256 help=
"input identifiers, e.g., --id visit=123 ccd=4")
257 self.add_argument(
"--calibId", nargs=
"*", action=CalibIdAction, default={},
258 help=
"identifiers for calib, e.g., --calibId version=1",
259 metavar=
"KEY=VALUE1[^VALUE2[^VALUE3...]")
264 Checks that the "--calibId" provided works. 266 namespace = ArgumentParser.parse_args(self, *args, **kwargs)
268 keys = namespace.butler.getKeys(self.
calibName)
270 for name, value
in namespace.calibId.items():
273 "%s is not a relevant calib identifier key (%s)" % (name, keys))
274 parsed[name] = keys[name](value)
275 namespace.calibId = parsed
281 """Configuration for constructing calibs""" 282 clobber = Field(dtype=bool, default=
True,
283 doc=
"Clobber existing processed images?")
284 isr = ConfigurableField(target=IsrTask, doc=
"ISR configuration")
285 dateObs = Field(dtype=str, default=
"dateObs",
286 doc=
"Key for observation date in exposure registry")
287 dateCalib = Field(dtype=str, default=
"calibDate",
288 doc=
"Key for calib date in calib registry")
289 filter = Field(dtype=str, default=
"filter",
290 doc=
"Key for filter name in exposure/calib registries")
291 combination = ConfigurableField(
292 target=CalibCombineTask, doc=
"Calib combination configuration")
293 ccdKeys = ListField(dtype=str, default=[
294 "ccd"], doc=
"DataId keywords specifying a CCD")
295 visitKeys = ListField(dtype=str, default=[
296 "visit"], doc=
"DataId keywords specifying a visit")
297 calibKeys = ListField(dtype=str, default=[],
298 doc=
"DataId keywords specifying a calibration")
301 self.
isr.doWrite =
False 305 """Get parsed values into the CalibTask.run""" 308 return [dict(expRefList=parsedCmd.id.refList, butler=parsedCmd.butler, calibId=parsedCmd.calibId)]
311 """Call the Task with the kwargs from getTargetList""" 312 task = self.TaskClass(config=self.config, log=self.log)
315 result = task.run(**args)
318 result = task.run(**args)
319 except Exception
as e:
322 task.log.fatal(
"Failed: %s" % e)
323 traceback.print_exc(file=sys.stderr)
325 if self.doReturnResults:
327 exitStatus=exitStatus,
329 metadata=task.metadata,
334 exitStatus=exitStatus,
338 """!Base class for constructing calibs. 340 This should be subclassed for each of the required calib types. 341 The subclass should be sure to define the following class variables: 342 * _DefaultName: default name of the task, used by CmdLineTask 343 * calibName: name of the calibration data set in the butler 344 The subclass may optionally set: 345 * filterName: filter name to give the resultant calib 347 ConfigClass = CalibConfig
348 RunnerClass = CalibTaskRunner
355 BatchPoolTask.__init__(self, *args, **kwargs)
356 self.makeSubtask(
"isr")
357 self.makeSubtask(
"combination")
361 numCcds = len(parsedCmd.butler.get(
"camera"))
363 parsedCmd)[0][
'expRefList'])
364 numCycles = int(numCcds/float(numCores) + 0.5)
365 return time*numExps*numCycles
368 def _makeArgumentParser(cls, *args, **kwargs):
369 kwargs.pop(
"doBatch",
False)
372 def run(self, expRefList, butler, calibId):
373 """!Construct a calib from a list of exposure references 375 This is the entry point, called by the TaskRunner.__call__ 377 Only the master node executes this method. 379 @param expRefList List of data references at the exposure level 380 @param butler Data butler 381 @param calibId Identifier dict for calib 383 for expRef
in expRefList:
384 self.
addMissingKeys(expRef.dataId, butler, self.config.ccdKeys,
'raw')
388 expRefList, level=
"sensor", ccdKeys=self.config.ccdKeys)
391 outputIdItemList = list(outputId.items())
392 for ccdName
in ccdIdLists:
393 dataId = dict([(k, ccdName[i])
for i, k
in enumerate(self.config.ccdKeys)])
394 dataId.update(outputIdItemList)
396 dataId.update(outputIdItemList)
399 butler.get(self.
calibName +
"_filename", dataId)
400 except Exception
as e:
402 "Unable to determine output filename \"%s_filename\" from %s: %s" %
406 pool.storeSet(butler=butler)
412 scales = self.
scale(ccdIdLists, data)
418 """!Generate the data identifier for the output calib 420 The mean date and the common filter are included, using keywords 421 from the configuration. The CCD-specific part is not included 422 in the data identifier. 424 @param expRefList List of data references at exposure level 425 @param calibId Data identifier elements for the calib provided by the user 426 @return data identifier 430 for expRef
in expRefList:
431 butler = expRef.getButler()
432 dataId = expRef.dataId
434 midTime += self.
getMjd(butler, dataId)
437 if filterName
is None:
438 filterName = thisFilter
439 elif filterName != thisFilter:
440 raise RuntimeError(
"Filter mismatch for %s: %s vs %s" % (
441 dataId, thisFilter, filterName))
443 midTime /= len(expRefList)
444 date = str(dafBase.DateTime(
445 midTime, dafBase.DateTime.MJD).toPython().date())
447 outputId = {self.config.filter: filterName,
448 self.config.dateCalib: date}
449 outputId.update(calibId)
452 def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC):
453 """Determine the Modified Julian Date (MJD; in TAI) from a data identifier""" 454 if self.config.dateObs
in dataId:
455 dateObs = dataId[self.config.dateObs]
457 dateObs = butler.queryMetadata(
'raw', [self.config.dateObs], dataId)[0]
458 if "T" not in dateObs:
459 dateObs = dateObs +
"T12:00:00.0Z" 460 elif not dateObs.endswith(
"Z"):
463 return dafBase.DateTime(dateObs, timescale).get(dafBase.DateTime.MJD)
466 """Determine the filter from a data identifier""" 467 filt = butler.queryMetadata(
'raw', [self.config.filter], dataId)[0]
471 if calibName
is None:
474 if missingKeys
is None:
475 missingKeys = set(butler.getKeys(calibName).keys()) - set(dataId.keys())
477 for k
in missingKeys:
479 v = butler.queryMetadata(
'raw', [k], dataId)
480 except Exception
as e:
489 raise RuntimeError(
"No unique lookup for %s: %s" % (k, v))
492 """!Update the metadata from the VisitInfo 494 \param calibImage The image whose metadata is to be set 495 \param exposureTime The exposure time for the image 496 \param darkTime The time since the last read (default: exposureTime) 500 darkTime = exposureTime
502 visitInfo = afwImage.makeVisitInfo(exposureTime=exposureTime, darkTime=darkTime, **kwargs)
503 md = calibImage.getMetadata()
505 afwImage.setVisitInfoMetadata(md, visitInfo)
508 """!Scatter the processing among the nodes 510 We scatter each CCD independently (exposures aren't grouped together), 511 to make full use of all available processors. This necessitates piecing 512 everything back together in the same format as ccdIdLists afterwards. 514 Only the master node executes this method. 516 @param pool Process pool 517 @param ccdIdLists Dict of data identifier lists for each CCD name 518 @return Dict of lists of returned data for each CCD name 520 dataIdList = sum(ccdIdLists.values(), [])
521 self.log.info(
"Scatter processing")
523 resultList = pool.map(self.
process, dataIdList)
526 data = dict((ccdName, [
None] * len(expList))
527 for ccdName, expList
in ccdIdLists.items())
528 indices = dict(sum([[(tuple(dataId.values())
if dataId
is not None else None, (ccdName, expNum))
529 for expNum, dataId
in enumerate(expList)]
530 for ccdName, expList
in ccdIdLists.items()], []))
531 for dataId, result
in zip(dataIdList, resultList):
534 ccdName, expNum = indices[tuple(dataId.values())]
535 data[ccdName][expNum] = result
539 def process(self, cache, ccdId, outputName="postISRCCD"):
540 """!Process a CCD, specified by a data identifier 542 After processing, optionally returns a result (produced by 543 the 'processResult' method) calculated from the processed 544 exposure. These results will be gathered by the master node, 545 and is a means for coordinated scaling of all CCDs for flats, 548 Only slave nodes execute this method. 550 @param cache Process pool cache 551 @param ccdId Data identifier for CCD 552 @param outputName Output dataset name for butler 553 @return result from 'processResult' 556 self.log.warn(
"Null identifier received on %s" % NODE)
559 if self.config.clobber
or not sensorRef.datasetExists(outputName):
560 self.log.info(
"Processing %s on %s" % (ccdId, NODE))
563 except Exception
as e:
564 self.log.warn(
"Unable to process %s: %s" % (ccdId, e))
570 "Using previously persisted processed exposure for %s" % (sensorRef.dataId,))
571 exposure = sensorRef.get(outputName, immediate=
True)
575 """Process a single CCD, specified by a data reference 577 Generally, this simply means doing ISR. 579 Only slave nodes execute this method. 581 return self.isr.runDataRef(dataRef).exposure
584 """!Write the processed CCD 586 We need to write these out because we can't hold them all in 589 Only slave nodes execute this method. 591 @param dataRef Data reference 592 @param exposure CCD exposure to write 593 @param outputName Output dataset name for butler. 595 dataRef.put(exposure, outputName)
598 """Extract processing results from a processed exposure 600 This method generates what is gathered by the master node. 601 This can be a background measurement or similar for scaling 602 flat-fields. It must be picklable! 604 Only slave nodes execute this method. 609 """!Determine scaling across CCDs and exposures 611 This is necessary mainly for flats, so as to determine a 612 consistent scaling across the entire focal plane. This 613 implementation is simply a placeholder. 615 Only the master node executes this method. 617 @param ccdIdLists Dict of data identifier lists for each CCD tuple 618 @param data Dict of lists of returned data for each CCD tuple 619 @return dict of Struct(ccdScale: scaling for CCD, 620 expScales: scaling for each exposure 623 self.log.info(
"Scale on %s" % NODE)
624 return dict((name, Struct(ccdScale=
None, expScales=[
None] * len(ccdIdLists[name])))
625 for name
in ccdIdLists)
628 """!Scatter the combination of exposures across multiple nodes 630 In this case, we can only scatter across as many nodes as 633 Only the master node executes this method. 635 @param pool Process pool 636 @param outputId Output identifier (exposure part only) 637 @param ccdIdLists Dict of data identifier lists for each CCD name 638 @param scales Dict of structs with scales, for each CCD name 640 self.log.info(
"Scatter combination")
641 data = [Struct(ccdName=ccdName, ccdIdList=ccdIdLists[ccdName], scales=scales[ccdName])
for 642 ccdName
in ccdIdLists]
643 pool.map(self.
combine, data, outputId)
646 """!Combine multiple exposures of a particular CCD and write the output 648 Only the slave nodes execute this method. 650 @param cache Process pool cache 651 @param struct Parameters for the combination, which has the following components: 652 * ccdName Name tuple for CCD 653 * ccdIdList List of data identifiers for combination 654 * scales Scales to apply (expScales are scalings for each exposure, 655 ccdScale is final scale for combined image) 656 @param outputId Data identifier for combined image (exposure part only) 659 fullOutputId = {k: struct.ccdName[i]
for i, k
in enumerate(self.config.ccdKeys)}
660 fullOutputId.update(outputId)
662 fullOutputId.update(outputId)
663 outputId = fullOutputId
666 dataRefList = [
getDataRef(cache.butler, dataId)
if dataId
is not None else None for 667 dataId
in struct.ccdIdList]
668 self.log.info(
"Combining %s on %s" % (outputId, NODE))
669 calib = self.combination.
run(dataRefList, expScales=struct.scales.expScales,
670 finalScale=struct.scales.ccdScale)
672 if not hasattr(calib,
"getMetadata"):
673 if hasattr(calib,
"getVariance"):
674 calib = afwImage.makeExposure(calib)
676 calib = afwImage.DecoratedImageF(calib.getImage())
681 struct.ccdIdList, outputId)
685 self.
write(cache.butler, calib, outputId)
688 """!Record metadata including the inputs and creation details 690 This metadata will go into the FITS header. 692 @param butler Data butler 693 @param calib Combined calib exposure. 694 @param dataIdList List of data identifiers for calibration inputs 695 @param outputId Data identifier for output 697 header = calib.getMetadata()
701 now = time.localtime()
702 header.add(
"CALIB_CREATION_DATE", time.strftime(
"%Y-%m-%d", now))
703 header.add(
"CALIB_CREATION_TIME", time.strftime(
"%X %Z", now))
705 header.add(
"DATE-OBS",
"%sT00:00:00.00" % outputId[self.config.dateCalib])
708 visits = [str(
dictToTuple(dataId, self.config.visitKeys))
for dataId
in dataIdList
if 710 for i, v
in enumerate(sorted(set(visits))):
711 header.add(
"CALIB_INPUT_%d" % (i,), v)
713 header.add(
"CALIB_ID",
" ".join(
"%s=%s" % (key, value)
714 for key, value
in outputId.items()))
718 """Interpolate over NANs in the combined image 720 NANs can result from masked areas on the CCD. We don't want them getting 721 into our science images, so we replace them with the median of the image. 723 if hasattr(image,
"getMaskedImage"):
725 image = image.getMaskedImage().getImage()
726 if hasattr(image,
"getImage"):
727 image = image.getImage()
728 array = image.getArray()
729 bad = np.isnan(array)
730 array[bad] = np.median(array[np.logical_not(bad)])
732 def write(self, butler, exposure, dataId):
733 """!Write the final combined calib 735 Only the slave nodes execute this method 737 @param butler Data butler 738 @param exposure CCD exposure to write 739 @param dataId Data identifier for output 741 self.log.info(
"Writing %s on %s" % (dataId, NODE))
742 butler.put(exposure, self.
calibName, dataId)
746 """Configuration for bias construction. 748 No changes required compared to the base class, but 749 subclassed for distinction. 754 class BiasTask(CalibTask):
755 """Bias construction""" 756 ConfigClass = BiasConfig
757 _DefaultName =
"bias" 764 """Overrides to apply for bias construction""" 765 config.isr.doBias =
False 766 config.isr.doDark =
False 767 config.isr.doFlat =
False 768 config.isr.doFringe =
False 772 """Configuration for dark construction""" 773 doRepair = Field(dtype=bool, default=
True, doc=
"Repair artifacts?")
774 psfFwhm = Field(dtype=float, default=3.0, doc=
"Repair PSF FWHM (pixels)")
775 psfSize = Field(dtype=int, default=21, doc=
"Repair PSF size (pixels)")
776 crGrow = Field(dtype=int, default=2, doc=
"Grow radius for CR (pixels)")
777 repair = ConfigurableField(
778 target=RepairTask, doc=
"Task to repair artifacts")
781 CalibConfig.setDefaults(self)
788 The only major difference from the base class is a cosmic-ray 789 identification stage, and dividing each image by the dark time 790 to generate images of the dark rate. 792 ConfigClass = DarkConfig
793 _DefaultName =
"dark" 798 CalibTask.__init__(self, *args, **kwargs)
799 self.makeSubtask(
"repair")
803 """Overrides to apply for dark construction""" 804 config.isr.doDark =
False 805 config.isr.doFlat =
False 806 config.isr.doFringe =
False 809 """Process a single CCD 811 Besides the regular ISR, also masks cosmic-rays and divides each 812 processed image by the dark time to generate images of the dark rate. 813 The dark time is provided by the 'getDarkTime' method. 815 exposure = CalibTask.processSingle(self, sensorRef)
817 if self.config.doRepair:
818 psf = measAlg.DoubleGaussianPsf(self.config.psfSize, self.config.psfSize,
819 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
821 self.repair.
run(exposure, keepCRs=
False)
822 if self.config.crGrow > 0:
823 mask = exposure.getMaskedImage().getMask().clone()
824 mask &= mask.getPlaneBitMask(
"CR")
825 fpSet = afwDet.FootprintSet(
826 mask, afwDet.Threshold(0.5))
827 fpSet = afwDet.FootprintSet(fpSet, self.config.crGrow,
True)
828 fpSet.setMask(exposure.getMaskedImage().getMask(),
"CR")
830 mi = exposure.getMaskedImage()
835 """Retrieve the dark time for an exposure""" 836 darkTime = exposure.getInfo().getVisitInfo().
getDarkTime()
837 if not np.isfinite(darkTime):
838 raise RuntimeError(
"Non-finite darkTime")
843 """Configuration for flat construction""" 844 iterations = Field(dtype=int, default=10,
845 doc=
"Number of iterations for scale determination")
846 stats = ConfigurableField(target=CalibStatsTask,
847 doc=
"Background statistics configuration")
853 The principal change from the base class involves gathering the background 854 values from each image and using them to determine the scalings for the final 857 ConfigClass = FlatConfig
858 _DefaultName =
"flat" 863 """Overrides for flat construction""" 864 config.isr.doFlat =
False 865 config.isr.doFringe =
False 868 CalibTask.__init__(self, *args, **kwargs)
869 self.makeSubtask(
"stats")
872 return self.stats.
run(exposure)
875 """Determine the scalings for the final combination 877 We have a matrix B_ij = C_i E_j, where C_i is the relative scaling 878 of one CCD to all the others in an exposure, and E_j is the scaling 879 of the exposure. We convert everything to logarithms so we can work 880 with a linear system. We determine the C_i and E_j from B_ij by iteration, 881 under the additional constraint that the average CCD scale is unity. 883 This algorithm comes from Eugene Magnier and Pan-STARRS. 885 assert len(ccdIdLists.values()) > 0,
"No successful CCDs" 886 lengths = set([len(expList)
for expList
in ccdIdLists.values()])
888 lengths) == 1,
"Number of successful exposures for each CCD differs" 889 assert tuple(lengths)[0] > 0,
"No successful exposures" 891 indices = dict((name, i)
for i, name
in enumerate(ccdIdLists))
892 bgMatrix = np.array([[0.0] * len(expList)
893 for expList
in ccdIdLists.values()])
894 for name
in ccdIdLists:
897 d
if d
is not None else np.nan
for d
in data[name]]
899 numpyPrint = np.get_printoptions()
900 np.set_printoptions(threshold=
'nan')
901 self.log.info(
"Input backgrounds: %s" % bgMatrix)
904 numCcds = len(ccdIdLists)
905 numExps = bgMatrix.shape[1]
907 bgMatrix = np.log(bgMatrix)
908 bgMatrix = np.ma.masked_array(bgMatrix, np.isnan(bgMatrix))
910 compScales = np.zeros(numCcds)
911 expScales = np.array(
912 [(bgMatrix[:, i0] - compScales).mean()
for i0
in range(numExps)])
914 for iterate
in range(self.config.iterations):
915 compScales = np.array(
916 [(bgMatrix[i1, :] - expScales).mean()
for i1
in range(numCcds)])
917 expScales = np.array(
918 [(bgMatrix[:, i2] - compScales).mean()
for i2
in range(numExps)])
920 avgScale = np.average(np.exp(compScales))
921 compScales -= np.log(avgScale)
922 self.log.debug(
"Iteration %d exposure scales: %s",
923 iterate, np.exp(expScales))
924 self.log.debug(
"Iteration %d component scales: %s",
925 iterate, np.exp(compScales))
927 expScales = np.array(
928 [(bgMatrix[:, i3] - compScales).mean()
for i3
in range(numExps)])
930 if np.any(np.isnan(expScales)):
931 raise RuntimeError(
"Bad exposure scales: %s --> %s" %
932 (bgMatrix, expScales))
934 expScales = np.exp(expScales)
935 compScales = np.exp(compScales)
937 self.log.info(
"Exposure scales: %s" % expScales)
938 self.log.info(
"Component relative scaling: %s" % compScales)
939 np.set_printoptions(**numpyPrint)
941 return dict((ccdName, Struct(ccdScale=compScales[indices[ccdName]], expScales=expScales))
942 for ccdName
in ccdIdLists)
946 """Configuration for fringe construction""" 947 stats = ConfigurableField(target=CalibStatsTask,
948 doc=
"Background statistics configuration")
949 subtractBackground = ConfigurableField(target=measAlg.SubtractBackgroundTask,
950 doc=
"Background configuration")
951 detection = ConfigurableField(
952 target=measAlg.SourceDetectionTask, doc=
"Detection configuration")
953 detectSigma = Field(dtype=float, default=1.0,
954 doc=
"Detection PSF gaussian sigma")
958 """Fringe construction task 960 The principal change from the base class is that the images are 961 background-subtracted and rescaled by the background. 963 XXX This is probably not right for a straight-up combination, as we 964 are currently doing, since the fringe amplitudes need not scale with 967 XXX Would like to have this do PCA and generate multiple images, but 968 that will take a bit of work with the persistence code. 970 ConfigClass = FringeConfig
971 _DefaultName =
"fringe" 976 """Overrides for fringe construction""" 977 config.isr.doFringe =
False 980 CalibTask.__init__(self, *args, **kwargs)
981 self.makeSubtask(
"detection")
982 self.makeSubtask(
"stats")
983 self.makeSubtask(
"subtractBackground")
986 """Subtract the background and normalise by the background level""" 987 exposure = CalibTask.processSingle(self, sensorRef)
988 bgLevel = self.stats.
run(exposure)
989 self.subtractBackground.
run(exposure)
990 mi = exposure.getMaskedImage()
992 footprintSets = self.detection.detectFootprints(
993 exposure, sigma=self.config.detectSigma)
994 mask = exposure.getMaskedImage().getMask()
995 detected = 1 << mask.addMaskPlane(
"DETECTED")
996 for fpSet
in (footprintSets.positive, footprintSets.negative):
997 if fpSet
is not None:
998 afwDet.setMaskFromFootprintList(
999 mask, fpSet.getFootprints(), detected)
def applyOverrides(cls, config)
def getFilter(self, butler, dataId)
def __init__(self, args, kwargs)
def processWrite(self, dataRef, exposure, outputName="postISRCCD")
Write the processed CCD.
def scatterCombine(self, pool, outputId, ccdIdLists, scales)
Scatter the combination of exposures across multiple nodes.
def run(self, exposureOrImage)
Measure a particular statistic on an image (of some sort).
def processResult(self, exposure)
def checksum(obj, header=None, sumType="MD5")
Calculate a checksum of an object.
def getCcdIdListFromExposures(expRefList, level="sensor", ccdKeys=["ccd"])
Determine a list of CCDs from exposure references.
def __init__(self, calibName, args, kwargs)
def applyScale(self, exposure, scale=None)
def processSingle(self, sensorRef)
def applyOverrides(cls, config)
def run(self, expRefList, butler, calibId)
Construct a calib from a list of exposure references.
def processSingle(self, sensorRef)
def __call__(self, parser, namespace, values, option_string)
def processSingle(self, dataRef)
def dictToTuple(dict_, keys)
Return a tuple of specific values from a dict.
def interpolateNans(self, image)
def getDataRef(butler, dataId, datasetType="raw")
def updateMetadata(self, calibImage, exposureTime, darkTime=None, kwargs)
Update the metadata from the VisitInfo.
def scale(self, ccdIdLists, data)
def parse_args(self, args, kwargs)
def combine(self, target, imageList, stats)
Combine multiple images.
def getMjd(self, butler, dataId, timescale=dafBase.DateTime.UTC)
def process(self, cache, ccdId, outputName="postISRCCD")
Process a CCD, specified by a data identifier.
def getDarkTime(self, exposure)
def combine(self, cache, struct, outputId)
Combine multiple exposures of a particular CCD and write the output.
def write(self, butler, exposure, dataId)
Write the final combined calib.
def recordCalibInputs(self, butler, calib, dataIdList, outputId)
Record metadata including the inputs and creation details.
def run(self, sensorRefList, expScales=None, finalScale=None, inputName="postISRCCD")
Combine calib images for a single sensor.
def __init__(self, args, kwargs)
def __init__(self, args, kwargs)
def applyOverrides(cls, config)
def getOutputId(self, expRefList, calibId)
Generate the data identifier for the output calib.
def processResult(self, exposure)
def __init__(self, args, kwargs)
def scale(self, ccdIdLists, data)
Determine scaling across CCDs and exposures.
def addMissingKeys(self, dataId, butler, missingKeys=None, calibName=None)
def getDimensions(self, sensorRefList, inputName="postISRCCD")
def getTargetList(parsedCmd, kwargs)
def __init__(self, args, kwargs)
Base class for constructing calibs.
def applyOverrides(cls, config)
def scatterProcess(self, pool, ccdIdLists)
Scatter the processing among the nodes.
def batchWallTime(cls, time, parsedCmd, numCores)