Coverage for python/lsst/cp/pipe/cpCombine.py: 24%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import time
24import lsst.pex.config as pexConfig
25import lsst.pipe.base as pipeBase
26import lsst.pipe.base.connectionTypes as cT
27import lsst.afw.math as afwMath
28import lsst.afw.image as afwImage
30from lsst.ip.isr.vignette import maskVignettedRegion
32from astro_metadata_translator import merge_headers, ObservationGroup
33from astro_metadata_translator.serialize import dates_to_fits
36__all__ = ['CalibStatsConfig', 'CalibStatsTask',
37 'CalibCombineConfig', 'CalibCombineConnections', 'CalibCombineTask',
38 'CalibCombineByFilterConfig', 'CalibCombineByFilterConnections', 'CalibCombineByFilterTask']
41# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
42class CalibStatsConfig(pexConfig.Config):
43 """Parameters controlling the measurement of background
44 statistics.
45 """
47 stat = pexConfig.Field(
48 dtype=str,
49 default='MEANCLIP',
50 doc="Statistic name to use to estimate background (from `~lsst.afw.math.Property`)",
51 )
52 clip = pexConfig.Field(
53 dtype=float,
54 default=3.0,
55 doc="Clipping threshold for background",
56 )
57 nIter = pexConfig.Field(
58 dtype=int,
59 default=3,
60 doc="Clipping iterations for background",
61 )
62 mask = pexConfig.ListField(
63 dtype=str,
64 default=["DETECTED", "BAD", "NO_DATA"],
65 doc="Mask planes to reject",
66 )
69class CalibStatsTask(pipeBase.Task):
70 """Measure statistics on the background
72 This can be useful for scaling the background, e.g., for flats and
73 fringe frames.
74 """
76 ConfigClass = CalibStatsConfig
78 def run(self, exposureOrImage):
79 """Measure a particular statistic on an image (of some sort).
81 Parameters
82 ----------
83 exposureOrImage : `lsst.afw.image.Exposure`,
84 `lsst.afw.image.MaskedImage`, or
85 `lsst.afw.image.Image`
86 Exposure or image to calculate statistics on.
88 Returns
89 -------
90 results : float
91 Resulting statistic value.
92 """
93 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
94 afwImage.Mask.getPlaneBitMask(self.config.mask))
95 try:
96 image = exposureOrImage.getMaskedImage()
97 except Exception:
98 try:
99 image = exposureOrImage.getImage()
100 except Exception:
101 image = exposureOrImage
102 statType = afwMath.stringToStatisticsProperty(self.config.stat)
103 return afwMath.makeStatistics(image, statType, stats).getValue()
106class CalibCombineConnections(pipeBase.PipelineTaskConnections,
107 dimensions=("instrument", "detector")):
108 inputExps = cT.Input(
109 name="cpInputs",
110 doc="Input pre-processed exposures to combine.",
111 storageClass="Exposure",
112 dimensions=("instrument", "detector", "exposure"),
113 multiple=True,
114 )
115 inputScales = cT.Input(
116 name="cpScales",
117 doc="Input scale factors to use.",
118 storageClass="StructuredDataDict",
119 dimensions=("instrument", ),
120 multiple=False,
121 )
123 outputData = cT.Output(
124 name="cpProposal",
125 doc="Output combined proposed calibration to be validated and certified..",
126 storageClass="ExposureF",
127 dimensions=("instrument", "detector"),
128 isCalibration=True,
129 )
131 def __init__(self, *, config=None):
132 super().__init__(config=config)
134 if config and config.exposureScaling != 'InputList':
135 self.inputs.discard("inputScales")
138# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
139class CalibCombineConfig(pipeBase.PipelineTaskConfig,
140 pipelineConnections=CalibCombineConnections):
141 """Configuration for combining calib exposures.
142 """
144 calibrationType = pexConfig.Field(
145 dtype=str,
146 default="calibration",
147 doc="Name of calibration to be generated.",
148 )
150 exposureScaling = pexConfig.ChoiceField(
151 dtype=str,
152 allowed={
153 "Unity": "Do not scale inputs. Scale factor is 1.0.",
154 "ExposureTime": "Scale inputs by their exposure time.",
155 "DarkTime": "Scale inputs by their dark time.",
156 "MeanStats": "Scale inputs based on their mean values.",
157 "InputList": "Scale inputs based on a list of values.",
158 },
159 default="Unity",
160 doc="Scaling to be applied to each input exposure.",
161 )
162 scalingLevel = pexConfig.ChoiceField(
163 dtype=str,
164 allowed={
165 "DETECTOR": "Scale by detector.",
166 "AMP": "Scale by amplifier.",
167 },
168 default="DETECTOR",
169 doc="Region to scale.",
170 )
171 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
172 dtype=int,
173 default=5,
174 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
175 )
177 doVignette = pexConfig.Field(
178 dtype=bool,
179 default=False,
180 doc="Copy vignette polygon to output and censor vignetted pixels?"
181 )
183 mask = pexConfig.ListField(
184 dtype=str,
185 default=["SAT", "DETECTED", "INTRP"],
186 doc="Mask planes to respect",
187 )
188 combine = pexConfig.Field(
189 dtype=str,
190 default='MEANCLIP',
191 doc="Statistic name to use for combination (from `~lsst.afw.math.Property`)",
192 )
193 clip = pexConfig.Field(
194 dtype=float,
195 default=3.0,
196 doc="Clipping threshold for combination",
197 )
198 nIter = pexConfig.Field(
199 dtype=int,
200 default=3,
201 doc="Clipping iterations for combination",
202 )
203 stats = pexConfig.ConfigurableField(
204 target=CalibStatsTask,
205 doc="Background statistics configuration",
206 )
209class CalibCombineTask(pipeBase.PipelineTask,
210 pipeBase.CmdLineTask):
211 """Task to combine calib exposures."""
213 ConfigClass = CalibCombineConfig
214 _DefaultName = 'cpCombine'
216 def __init__(self, **kwargs):
217 super().__init__(**kwargs)
218 self.makeSubtask("stats")
220 def runQuantum(self, butlerQC, inputRefs, outputRefs):
221 inputs = butlerQC.get(inputRefs)
223 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
224 inputs['inputDims'] = dimensions
226 outputs = self.run(**inputs)
227 butlerQC.put(outputs, outputRefs)
229 def run(self, inputExps, inputScales=None, inputDims=None):
230 """Combine calib exposures for a single detector.
232 Parameters
233 ----------
234 inputExps : `list` [`lsst.afw.image.Exposure`]
235 Input list of exposures to combine.
236 inputScales : `dict` [`dict` [`dict` [`float`]]], optional
237 Dictionary of scales, indexed by detector (`int`),
238 amplifier (`int`), and exposure (`int`). Used for
239 'inputExps' scaling.
240 inputDims : `list` [`dict`]
241 List of dictionaries of input data dimensions/values.
242 Each list entry should contain:
244 ``"exposure"``
245 exposure id value (`int`)
246 ``"detector"``
247 detector id value (`int`)
249 Returns
250 -------
251 results : `lsst.pipe.base.Struct`
252 The results struct containing:
254 ``combinedExp``
255 Final combined exposure generated from the inputs
256 (`lsst.afw.image.Exposure`).
258 Raises
259 ------
260 RuntimeError
261 Raised if no input data is found. Also raised if
262 config.exposureScaling == InputList, and a necessary scale
263 was not found.
264 """
265 width, height = self.getDimensions(inputExps)
266 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
267 afwImage.Mask.getPlaneBitMask(self.config.mask))
268 numExps = len(inputExps)
269 if numExps < 1:
270 raise RuntimeError("No valid input data")
271 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
272 stats.setCalcErrorFromInputVariance(True)
274 # Check that all inputs either share the same detector (based
275 # on detId), or that no inputs have any detector.
276 detectorList = [exp.getDetector() for exp in inputExps]
277 if None in detectorList:
278 self.log.warning("Not all input detectors defined.")
279 detectorIds = [det.getId() if det is not None else None for det in detectorList]
280 detectorSerials = [det.getId() if det is not None else None for det in detectorList]
281 numDetectorIds = len(set(detectorIds))
282 numDetectorSerials = len(set(detectorSerials))
283 numDetectors = len(set([numDetectorIds, numDetectorSerials]))
284 if numDetectors != 1:
285 raise RuntimeError("Input data contains multiple detectors.")
287 inputDetector = inputExps[0].getDetector()
289 # Create output exposure for combined data.
290 combined = afwImage.MaskedImageF(width, height)
291 combinedExp = afwImage.makeExposure(combined)
293 # Apply scaling:
294 expScales = []
295 if inputDims is None:
296 inputDims = [dict() for i in inputExps]
298 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
299 scale = 1.0
300 if exp is None:
301 self.log.warning("Input %d is None (%s); unable to scale exp.", index, dims)
302 continue
304 if self.config.exposureScaling == "ExposureTime":
305 scale = exp.getInfo().getVisitInfo().getExposureTime()
306 elif self.config.exposureScaling == "DarkTime":
307 scale = exp.getInfo().getVisitInfo().getDarkTime()
308 elif self.config.exposureScaling == "MeanStats":
309 scale = self.stats.run(exp)
310 elif self.config.exposureScaling == "InputList":
311 visitId = dims.get('exposure', None)
312 detectorId = dims.get('detector', None)
313 if visitId is None or detectorId is None:
314 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
315 if detectorId not in inputScales['expScale']:
316 raise RuntimeError(f"Could not identify a scaling for input {index}"
317 f" detector {detectorId}")
319 if self.config.scalingLevel == "DETECTOR":
320 if visitId not in inputScales['expScale'][detectorId]:
321 raise RuntimeError(f"Could not identify a scaling for input {index}"
322 f"detector {detectorId} visit {visitId}")
323 scale = inputScales['expScale'][detectorId][visitId]
324 elif self.config.scalingLevel == 'AMP':
325 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
326 for amp in exp.getDetector()]
327 else:
328 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
329 elif self.config.exposureScaling == 'Unity':
330 scale = 1.0
331 else:
332 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
334 expScales.append(scale)
335 self.log.info("Scaling input %d by %s", index, scale)
336 self.applyScale(exp, scale)
338 self.combine(combined, inputExps, stats)
340 self.interpolateNans(combined)
342 if self.config.doVignette:
343 polygon = inputExps[0].getInfo().getValidPolygon()
344 maskVignettedRegion(combined, polygon=polygon, vignetteValue=0.0)
346 # Combine headers
347 self.combineHeaders(inputExps, combinedExp,
348 calibType=self.config.calibrationType, scales=expScales)
350 # Set the detector
351 combinedExp.setDetector(inputDetector)
353 # Return
354 return pipeBase.Struct(
355 outputData=combinedExp,
356 )
358 def getDimensions(self, expList):
359 """Get dimensions of the inputs.
361 Parameters
362 ----------
363 expList : `list` [`lsst.afw.image.Exposure`]
364 Exps to check the sizes of.
366 Returns
367 -------
368 width, height : `int`
369 Unique set of input dimensions.
370 """
371 dimList = [exp.getDimensions() for exp in expList if exp is not None]
372 return self.getSize(dimList)
374 def getSize(self, dimList):
375 """Determine a consistent size, given a list of image sizes.
377 Parameters
378 -----------
379 dimList : `list` [`tuple` [`int`, `int`]]
380 List of dimensions.
382 Raises
383 ------
384 RuntimeError
385 If input dimensions are inconsistent.
387 Returns
388 --------
389 width, height : `int`
390 Common dimensions.
391 """
392 dim = set((w, h) for w, h in dimList)
393 if len(dim) != 1:
394 raise RuntimeError("Inconsistent dimensions: %s" % dim)
395 return dim.pop()
397 def applyScale(self, exposure, scale=None):
398 """Apply scale to input exposure.
400 This implementation applies a flux scaling: the input exposure is
401 divided by the provided scale.
403 Parameters
404 ----------
405 exposure : `lsst.afw.image.Exposure`
406 Exposure to scale.
407 scale : `float` or `list` [`float`], optional
408 Constant scale to divide the exposure by.
409 """
410 if scale is not None:
411 mi = exposure.getMaskedImage()
412 if isinstance(scale, list):
413 for amp, ampScale in zip(exposure.getDetector(), scale):
414 ampIm = mi[amp.getBBox()]
415 ampIm /= ampScale
416 else:
417 mi /= scale
419 def combine(self, target, expList, stats):
420 """Combine multiple images.
422 Parameters
423 ----------
424 target : `lsst.afw.image.Exposure`
425 Output exposure to construct.
426 expList : `list` [`lsst.afw.image.Exposure`]
427 Input exposures to combine.
428 stats : `lsst.afw.math.StatisticsControl`
429 Control explaining how to combine the input images.
430 """
431 images = [img.getMaskedImage() for img in expList if img is not None]
432 combineType = afwMath.stringToStatisticsProperty(self.config.combine)
433 afwMath.statisticsStack(target, images, combineType, stats)
435 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
436 """Combine input headers to determine the set of common headers,
437 supplemented by calibration inputs.
439 Parameters
440 ----------
441 expList : `list` [`lsst.afw.image.Exposure`]
442 Input list of exposures to combine.
443 calib : `lsst.afw.image.Exposure`
444 Output calibration to construct headers for.
445 calibType : `str`, optional
446 OBSTYPE the output should claim.
447 scales : `list` [`float`], optional
448 Scale values applied to each input to record.
450 Returns
451 -------
452 header : `lsst.daf.base.PropertyList`
453 Constructed header.
454 """
455 # Header
456 header = calib.getMetadata()
457 header.set("OBSTYPE", calibType)
459 # Keywords we care about
460 comments = {"TIMESYS": "Time scale for all dates",
461 "DATE-OBS": "Start date of earliest input observation",
462 "MJD-OBS": "[d] Start MJD of earliest input observation",
463 "DATE-END": "End date of oldest input observation",
464 "MJD-END": "[d] End MJD of oldest input observation",
465 "MJD-AVG": "[d] MJD midpoint of all input observations",
466 "DATE-AVG": "Midpoint date of all input observations"}
468 # Creation date
469 now = time.localtime()
470 calibDate = time.strftime("%Y-%m-%d", now)
471 calibTime = time.strftime("%X %Z", now)
472 header.set("CALIB_CREATE_DATE", calibDate)
473 header.set("CALIB_CREATE_TIME", calibTime)
475 # Merge input headers
476 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
477 merged = merge_headers(inputHeaders, mode='drop')
478 for k, v in merged.items():
479 if k not in header:
480 md = expList[0].getMetadata()
481 comment = md.getComment(k) if k in md else None
482 header.set(k, v, comment=comment)
484 # Construct list of visits
485 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
486 for i, visit in enumerate(visitInfoList):
487 if visit is None:
488 continue
489 header.set("CPP_INPUT_%d" % (i,), visit.id)
490 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
491 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
492 if scales is not None:
493 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
495 # Not yet working: DM-22302
496 # Create an observation group so we can add some standard headers
497 # independent of the form in the input files.
498 # Use try block in case we are dealing with unexpected data headers
499 try:
500 group = ObservationGroup(visitInfoList, pedantic=False)
501 except Exception:
502 self.log.warning("Exception making an obs group for headers. Continuing.")
503 # Fall back to setting a DATE-OBS from the calibDate
504 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
505 comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
506 else:
507 oldest, newest = group.extremes()
508 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
510 for k, v in dateCards.items():
511 header.set(k, v, comment=comments.get(k, None))
513 return header
515 def interpolateNans(self, exp):
516 """Interpolate over NANs in the combined image.
518 NANs can result from masked areas on the CCD. We don't want
519 them getting into our science images, so we replace them with
520 the median of the image.
522 Parameters
523 ----------
524 exp : `lsst.afw.image.Exposure`
525 Exp to check for NaNs.
526 """
527 array = exp.getImage().getArray()
528 bad = np.isnan(array)
530 median = np.median(array[np.logical_not(bad)])
531 count = np.sum(np.logical_not(bad))
532 array[bad] = median
533 if count > 0:
534 self.log.warning("Found %s NAN pixels", count)
537# Create versions of the Connections, Config, and Task that support
538# filter constraints.
539class CalibCombineByFilterConnections(CalibCombineConnections,
540 dimensions=("instrument", "detector", "physical_filter")):
541 inputScales = cT.Input(
542 name="cpFilterScales",
543 doc="Input scale factors to use.",
544 storageClass="StructuredDataDict",
545 dimensions=("instrument", "physical_filter"),
546 multiple=False,
547 )
549 outputData = cT.Output(
550 name="cpFilterProposal",
551 doc="Output combined proposed calibration to be validated and certified.",
552 storageClass="ExposureF",
553 dimensions=("instrument", "detector", "physical_filter"),
554 isCalibration=True,
555 )
557 def __init__(self, *, config=None):
558 super().__init__(config=config)
560 if config and config.exposureScaling != 'InputList':
561 self.inputs.discard("inputScales")
564class CalibCombineByFilterConfig(CalibCombineConfig,
565 pipelineConnections=CalibCombineByFilterConnections):
566 pass
569class CalibCombineByFilterTask(CalibCombineTask):
570 """Task to combine calib exposures."""
572 ConfigClass = CalibCombineByFilterConfig
573 _DefaultName = 'cpFilterCombine'
574 pass