Coverage for python/lsst/cp/pipe/cpCombine.py: 25%
212 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-05 03:37 -0700
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-05 03:37 -0700
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import time
24import lsst.geom as geom
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.afw.math as afwMath
29import lsst.afw.image as afwImage
31from lsst.ip.isr.vignette import maskVignettedRegion
33from astro_metadata_translator import merge_headers, ObservationGroup
34from astro_metadata_translator.serialize import dates_to_fits
37__all__ = ["CalibStatsConfig", "CalibStatsTask",
38 "CalibCombineConfig", "CalibCombineConnections", "CalibCombineTask",
39 "CalibCombineByFilterConfig", "CalibCombineByFilterConnections", "CalibCombineByFilterTask"]
42# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
43class CalibStatsConfig(pexConfig.Config):
44 """Parameters controlling the measurement of background
45 statistics.
46 """
48 stat = pexConfig.Field(
49 dtype=str,
50 default="MEANCLIP",
51 doc="Statistic name to use to estimate background (from `~lsst.afw.math.Property`)",
52 )
53 clip = pexConfig.Field(
54 dtype=float,
55 default=3.0,
56 doc="Clipping threshold for background",
57 )
58 nIter = pexConfig.Field(
59 dtype=int,
60 default=3,
61 doc="Clipping iterations for background",
62 )
63 mask = pexConfig.ListField(
64 dtype=str,
65 default=["DETECTED", "BAD", "NO_DATA"],
66 doc="Mask planes to reject",
67 )
70class CalibStatsTask(pipeBase.Task):
71 """Measure statistics on the background
73 This can be useful for scaling the background, e.g., for flats and
74 fringe frames.
75 """
77 ConfigClass = CalibStatsConfig
79 def run(self, exposureOrImage):
80 """Measure a particular statistic on an image (of some sort).
82 Parameters
83 ----------
84 exposureOrImage : `lsst.afw.image.Exposure`,
85 `lsst.afw.image.MaskedImage`, or
86 `lsst.afw.image.Image`
87 Exposure or image to calculate statistics on.
89 Returns
90 -------
91 results : float
92 Resulting statistic value.
93 """
94 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
95 afwImage.Mask.getPlaneBitMask(self.config.mask))
96 try:
97 image = exposureOrImage.getMaskedImage()
98 except Exception:
99 try:
100 image = exposureOrImage.getImage()
101 except Exception:
102 image = exposureOrImage
103 statType = afwMath.stringToStatisticsProperty(self.config.stat)
104 return afwMath.makeStatistics(image, statType, stats).getValue()
107class CalibCombineConnections(pipeBase.PipelineTaskConnections,
108 dimensions=("instrument", "detector")):
109 inputExpHandles = cT.Input(
110 name="cpInputs",
111 doc="Input pre-processed exposures to combine.",
112 storageClass="Exposure",
113 dimensions=("instrument", "detector", "exposure"),
114 multiple=True,
115 deferLoad=True,
116 )
117 inputScales = cT.Input(
118 name="cpScales",
119 doc="Input scale factors to use.",
120 storageClass="StructuredDataDict",
121 dimensions=("instrument", ),
122 multiple=False,
123 )
125 outputData = cT.Output(
126 name="cpProposal",
127 doc="Output combined proposed calibration to be validated and certified..",
128 storageClass="ExposureF",
129 dimensions=("instrument", "detector"),
130 isCalibration=True,
131 )
133 def __init__(self, *, config=None):
134 super().__init__(config=config)
136 if config and config.exposureScaling != "InputList":
137 self.inputs.discard("inputScales")
140# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
141class CalibCombineConfig(pipeBase.PipelineTaskConfig,
142 pipelineConnections=CalibCombineConnections):
143 """Configuration for combining calib exposures.
144 """
146 calibrationType = pexConfig.Field(
147 dtype=str,
148 default="calibration",
149 doc="Name of calibration to be generated.",
150 )
152 exposureScaling = pexConfig.ChoiceField(
153 dtype=str,
154 allowed={
155 "Unity": "Do not scale inputs. Scale factor is 1.0.",
156 "ExposureTime": "Scale inputs by their exposure time.",
157 "DarkTime": "Scale inputs by their dark time.",
158 "MeanStats": "Scale inputs based on their mean values.",
159 "InputList": "Scale inputs based on a list of values.",
160 },
161 default="Unity",
162 doc="Scaling to be applied to each input exposure.",
163 )
164 scalingLevel = pexConfig.ChoiceField(
165 dtype=str,
166 allowed={
167 "DETECTOR": "Scale by detector.",
168 "AMP": "Scale by amplifier.",
169 },
170 default="DETECTOR",
171 doc="Region to scale.",
172 )
173 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
174 dtype=int,
175 default=5,
176 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
177 )
178 subregionSize = pexConfig.ListField(
179 dtype=int,
180 doc="Width, height of subregion size.",
181 length=2,
182 # This is 200 rows for all detectors smaller than 10k in width.
183 default=(10000, 200),
184 )
186 doVignette = pexConfig.Field(
187 dtype=bool,
188 default=False,
189 doc="Copy vignette polygon to output and censor vignetted pixels?"
190 )
192 mask = pexConfig.ListField(
193 dtype=str,
194 default=["SAT", "DETECTED", "INTRP"],
195 doc="Mask planes to respect",
196 )
197 combine = pexConfig.Field(
198 dtype=str,
199 default="MEANCLIP",
200 doc="Statistic name to use for combination (from `~lsst.afw.math.Property`)",
201 )
202 clip = pexConfig.Field(
203 dtype=float,
204 default=3.0,
205 doc="Clipping threshold for combination",
206 )
207 nIter = pexConfig.Field(
208 dtype=int,
209 default=3,
210 doc="Clipping iterations for combination",
211 )
212 stats = pexConfig.ConfigurableField(
213 target=CalibStatsTask,
214 doc="Background statistics configuration",
215 )
218class CalibCombineTask(pipeBase.PipelineTask,
219 pipeBase.CmdLineTask):
220 """Task to combine calib exposures."""
222 ConfigClass = CalibCombineConfig
223 _DefaultName = "cpCombine"
225 def __init__(self, **kwargs):
226 super().__init__(**kwargs)
227 self.makeSubtask("stats")
229 def runQuantum(self, butlerQC, inputRefs, outputRefs):
230 inputs = butlerQC.get(inputRefs)
232 dimensions = [expHandle.dataId.byName() for expHandle in inputRefs.inputExpHandles]
233 inputs["inputDims"] = dimensions
235 outputs = self.run(**inputs)
236 butlerQC.put(outputs, outputRefs)
238 def run(self, inputExpHandles, inputScales=None, inputDims=None):
239 """Combine calib exposures for a single detector.
241 Parameters
242 ----------
243 inputExpHandles : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
244 Input list of exposure handles to combine.
245 inputScales : `dict` [`dict` [`dict` [`float`]]], optional
246 Dictionary of scales, indexed by detector (`int`),
247 amplifier (`int`), and exposure (`int`). Used for
248 'inputExps' scaling.
249 inputDims : `list` [`dict`]
250 List of dictionaries of input data dimensions/values.
251 Each list entry should contain:
253 ``"exposure"``
254 exposure id value (`int`)
255 ``"detector"``
256 detector id value (`int`)
258 Returns
259 -------
260 results : `lsst.pipe.base.Struct`
261 The results struct containing:
263 ``outputData``
264 Final combined exposure generated from the inputs
265 (`lsst.afw.image.Exposure`).
267 Raises
268 ------
269 RuntimeError
270 Raised if no input data is found. Also raised if
271 config.exposureScaling == InputList, and a necessary scale
272 was not found.
273 """
274 width, height = self.getDimensions(inputExpHandles)
275 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
276 afwImage.Mask.getPlaneBitMask(self.config.mask))
277 numExps = len(inputExpHandles)
278 if numExps < 1:
279 raise RuntimeError("No valid input data")
280 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
281 stats.setCalcErrorFromInputVariance(True)
283 inputDetector = inputExpHandles[0].get(component="detector")
285 # Create output exposure for combined data.
286 combined = afwImage.MaskedImageF(width, height)
287 combinedExp = afwImage.makeExposure(combined)
289 # Apply scaling:
290 expScales = []
291 if inputDims is None:
292 inputDims = [dict() for i in inputExpHandles]
294 for index, (expHandle, dims) in enumerate(zip(inputExpHandles, inputDims)):
295 scale = 1.0
296 visitInfo = expHandle.get(component="visitInfo")
297 if self.config.exposureScaling == "ExposureTime":
298 scale = visitInfo.getExposureTime()
299 elif self.config.exposureScaling == "DarkTime":
300 scale = visitInfo.getDarkTime()
301 elif self.config.exposureScaling == "MeanStats":
302 # Note: there may a bug freeing memory here. TBD.
303 exp = expHandle.get()
304 scale = self.stats.run(exp)
305 del exp
306 elif self.config.exposureScaling == "InputList":
307 visitId = dims.get("exposure", None)
308 detectorId = dims.get("detector", None)
309 if visitId is None or detectorId is None:
310 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
311 if detectorId not in inputScales["expScale"]:
312 raise RuntimeError(f"Could not identify a scaling for input {index}"
313 f" detector {detectorId}")
315 if self.config.scalingLevel == "DETECTOR":
316 if visitId not in inputScales["expScale"][detectorId]:
317 raise RuntimeError(f"Could not identify a scaling for input {index}"
318 f"detector {detectorId} visit {visitId}")
319 scale = inputScales["expScale"][detectorId][visitId]
320 elif self.config.scalingLevel == "AMP":
321 scale = [inputScales["expScale"][detectorId][amp.getName()][visitId]
322 for amp in inputDetector]
323 else:
324 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
325 elif self.config.exposureScaling == "Unity":
326 scale = 1.0
327 else:
328 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
330 expScales.append(scale)
331 self.log.info("Scaling input %d by %s", index, scale)
333 self.combine(combinedExp, inputExpHandles, expScales, stats)
335 self.interpolateNans(combined)
337 if self.config.doVignette:
338 polygon = inputExpHandles[0].get(component="validPolygon")
339 maskVignettedRegion(combined, polygon=polygon, vignetteValue=0.0)
341 # Combine headers
342 self.combineHeaders(inputExpHandles, combinedExp,
343 calibType=self.config.calibrationType, scales=expScales)
345 # Set the detector
346 combinedExp.setDetector(inputDetector)
348 # Return
349 return pipeBase.Struct(
350 outputData=combinedExp,
351 )
353 def getDimensions(self, expHandleList):
354 """Get dimensions of the inputs.
356 Parameters
357 ----------
358 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
359 Exposure handles to check the sizes of.
361 Returns
362 -------
363 width, height : `int`
364 Unique set of input dimensions.
365 """
366 dimList = [expHandle.get(component="bbox").getDimensions() for expHandle in expHandleList]
368 return self.getSize(dimList)
370 def getSize(self, dimList):
371 """Determine a consistent size, given a list of image sizes.
373 Parameters
374 -----------
375 dimList : `list` [`tuple` [`int`, `int`]]
376 List of dimensions.
378 Raises
379 ------
380 RuntimeError
381 If input dimensions are inconsistent.
383 Returns
384 --------
385 width, height : `int`
386 Common dimensions.
387 """
388 dim = set((w, h) for w, h in dimList)
389 if len(dim) != 1:
390 raise RuntimeError("Inconsistent dimensions: %s" % dim)
391 return dim.pop()
393 def applyScale(self, exposure, scale=None):
394 """Apply scale to input exposure.
396 This implementation applies a flux scaling: the input exposure is
397 divided by the provided scale.
399 Parameters
400 ----------
401 exposure : `lsst.afw.image.Exposure`
402 Exposure to scale.
403 scale : `float` or `list` [`float`], optional
404 Constant scale to divide the exposure by.
405 """
406 if scale is not None:
407 mi = exposure.getMaskedImage()
408 if isinstance(scale, list):
409 for amp, ampScale in zip(exposure.getDetector(), scale):
410 ampIm = mi[amp.getBBox()]
411 ampIm /= ampScale
412 else:
413 mi /= scale
415 @staticmethod
416 def _subBBoxIter(bbox, subregionSize):
417 """Iterate over subregions of a bbox.
419 Parameters
420 ----------
421 bbox : `lsst.geom.Box2I`
422 Bounding box over which to iterate.
423 subregionSize: `lsst.geom.Extent2I`
424 Size of sub-bboxes.
426 Yields
427 ------
428 subBBox : `lsst.geom.Box2I`
429 Next sub-bounding box of size ``subregionSize`` or
430 smaller; each ``subBBox`` is contained within ``bbox``, so
431 it may be smaller than ``subregionSize`` at the edges of
432 ``bbox``, but it will never be empty.
433 """
434 if bbox.isEmpty():
435 raise RuntimeError("bbox %s is empty" % (bbox,))
436 if subregionSize[0] < 1 or subregionSize[1] < 1:
437 raise RuntimeError("subregionSize %s must be nonzero" % (subregionSize,))
439 for rowShift in range(0, bbox.getHeight(), subregionSize[1]):
440 for colShift in range(0, bbox.getWidth(), subregionSize[0]):
441 subBBox = geom.Box2I(bbox.getMin() + geom.Extent2I(colShift, rowShift), subregionSize)
442 subBBox.clip(bbox)
443 if subBBox.isEmpty():
444 raise RuntimeError("Bug: empty bbox! bbox=%s, subregionSize=%s, "
445 "colShift=%s, rowShift=%s" %
446 (bbox, subregionSize, colShift, rowShift))
447 yield subBBox
449 def combine(self, target, expHandleList, expScaleList, stats):
450 """Combine multiple images.
452 Parameters
453 ----------
454 target : `lsst.afw.image.Exposure`
455 Output exposure to construct.
456 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
457 Input exposure handles to combine.
458 expScaleList : `list` [`float`]
459 List of scales to apply to each input image.
460 stats : `lsst.afw.math.StatisticsControl`
461 Control explaining how to combine the input images.
462 """
463 combineType = afwMath.stringToStatisticsProperty(self.config.combine)
465 subregionSizeArr = self.config.subregionSize
466 subregionSize = geom.Extent2I(subregionSizeArr[0], subregionSizeArr[1])
467 for subBbox in self._subBBoxIter(target.getBBox(), subregionSize):
468 images = []
469 for expHandle, expScale in zip(expHandleList, expScaleList):
470 inputExp = expHandle.get(parameters={"bbox": subBbox})
471 self.applyScale(inputExp, expScale)
472 images.append(inputExp.getMaskedImage())
474 combinedSubregion = afwMath.statisticsStack(images, combineType, stats)
475 target.maskedImage.assign(combinedSubregion, subBbox)
477 def combineHeaders(self, expHandleList, calib, calibType="CALIB", scales=None):
478 """Combine input headers to determine the set of common headers,
479 supplemented by calibration inputs.
481 Parameters
482 ----------
483 expHandleList : `list` [`lsst.daf.butler.DeferredDatasetHandle`]
484 Input list of exposure handles to combine.
485 calib : `lsst.afw.image.Exposure`
486 Output calibration to construct headers for.
487 calibType : `str`, optional
488 OBSTYPE the output should claim.
489 scales : `list` [`float`], optional
490 Scale values applied to each input to record.
492 Returns
493 -------
494 header : `lsst.daf.base.PropertyList`
495 Constructed header.
496 """
497 # Header
498 header = calib.getMetadata()
499 header.set("OBSTYPE", calibType)
501 # Keywords we care about
502 comments = {"TIMESYS": "Time scale for all dates",
503 "DATE-OBS": "Start date of earliest input observation",
504 "MJD-OBS": "[d] Start MJD of earliest input observation",
505 "DATE-END": "End date of oldest input observation",
506 "MJD-END": "[d] End MJD of oldest input observation",
507 "MJD-AVG": "[d] MJD midpoint of all input observations",
508 "DATE-AVG": "Midpoint date of all input observations"}
510 # Creation date
511 now = time.localtime()
512 calibDate = time.strftime("%Y-%m-%d", now)
513 calibTime = time.strftime("%X %Z", now)
514 header.set("CALIB_CREATE_DATE", calibDate)
515 header.set("CALIB_CREATE_TIME", calibTime)
517 # Merge input headers
518 inputHeaders = [expHandle.get(component="metadata") for expHandle in expHandleList]
519 merged = merge_headers(inputHeaders, mode="drop")
521 # Scan the first header for items that were dropped due to
522 # conflict, and replace them.
523 for k, v in merged.items():
524 if k not in header:
525 md = inputHeaders[0]
526 comment = md.getComment(k) if k in md else None
527 header.set(k, v, comment=comment)
529 # Construct list of visits
530 visitInfoList = [expHandle.get(component="visitInfo") for expHandle in expHandleList]
531 for i, visit in enumerate(visitInfoList):
532 if visit is None:
533 continue
534 header.set("CPP_INPUT_%d" % (i,), visit.id)
535 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
536 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
537 if scales is not None:
538 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
540 # Not yet working: DM-22302
541 # Create an observation group so we can add some standard headers
542 # independent of the form in the input files.
543 # Use try block in case we are dealing with unexpected data headers
544 try:
545 group = ObservationGroup(visitInfoList, pedantic=False)
546 except Exception:
547 self.log.warning("Exception making an obs group for headers. Continuing.")
548 # Fall back to setting a DATE-OBS from the calibDate
549 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
550 comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
551 else:
552 oldest, newest = group.extremes()
553 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
555 for k, v in dateCards.items():
556 header.set(k, v, comment=comments.get(k, None))
558 return header
560 def interpolateNans(self, exp):
561 """Interpolate over NANs in the combined image.
563 NANs can result from masked areas on the CCD. We don't want
564 them getting into our science images, so we replace them with
565 the median of the image.
567 Parameters
568 ----------
569 exp : `lsst.afw.image.Exposure`
570 Exp to check for NaNs.
571 """
572 array = exp.getImage().getArray()
573 bad = np.isnan(array)
574 if np.any(bad):
575 median = np.median(array[np.logical_not(bad)])
576 count = np.sum(bad)
577 array[bad] = median
578 self.log.warning("Found and fixed %s NAN pixels", count)
581# Create versions of the Connections, Config, and Task that support
582# filter constraints.
583class CalibCombineByFilterConnections(CalibCombineConnections,
584 dimensions=("instrument", "detector", "physical_filter")):
585 inputScales = cT.Input(
586 name="cpFilterScales",
587 doc="Input scale factors to use.",
588 storageClass="StructuredDataDict",
589 dimensions=("instrument", "physical_filter"),
590 multiple=False,
591 )
593 outputData = cT.Output(
594 name="cpFilterProposal",
595 doc="Output combined proposed calibration to be validated and certified.",
596 storageClass="ExposureF",
597 dimensions=("instrument", "detector", "physical_filter"),
598 isCalibration=True,
599 )
601 def __init__(self, *, config=None):
602 super().__init__(config=config)
604 if config and config.exposureScaling != "InputList":
605 self.inputs.discard("inputScales")
608class CalibCombineByFilterConfig(CalibCombineConfig,
609 pipelineConnections=CalibCombineByFilterConnections):
610 pass
613class CalibCombineByFilterTask(CalibCombineTask):
614 """Task to combine calib exposures."""
616 ConfigClass = CalibCombineByFilterConfig
617 _DefaultName = "cpFilterCombine"
618 pass