Coverage for python/lsst/cp/pipe/cpCombine.py : 18%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import time
24import lsst.pex.config as pexConfig
25import lsst.pipe.base as pipeBase
26import lsst.pipe.base.connectionTypes as cT
27import lsst.afw.math as afwMath
28import lsst.afw.image as afwImage
30from lsst.geom import Point2D
31from lsst.log import Log
32from astro_metadata_translator import merge_headers, ObservationGroup
33from astro_metadata_translator.serialize import dates_to_fits
36# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
37class CalibStatsConfig(pexConfig.Config):
38 """Parameters controlling the measurement of background statistics.
39 """
40 stat = pexConfig.Field(
41 dtype=str,
42 default='MEANCLIP',
43 doc="Statistic name to use to estimate background (from lsst.afw.math)",
44 )
45 clip = pexConfig.Field(
46 dtype=float,
47 default=3.0,
48 doc="Clipping threshold for background",
49 )
50 nIter = pexConfig.Field(
51 dtype=int,
52 default=3,
53 doc="Clipping iterations for background",
54 )
55 mask = pexConfig.ListField(
56 dtype=str,
57 default=["DETECTED", "BAD", "NO_DATA"],
58 doc="Mask planes to reject",
59 )
62class CalibStatsTask(pipeBase.Task):
63 """Measure statistics on the background
65 This can be useful for scaling the background, e.g., for flats and fringe frames.
66 """
67 ConfigClass = CalibStatsConfig
69 def run(self, exposureOrImage):
70 """Measure a particular statistic on an image (of some sort).
72 Parameters
73 ----------
74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75 Exposure or image to calculate statistics on.
77 Returns
78 -------
79 results : float
80 Resulting statistic value.
81 """
82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83 afwImage.Mask.getPlaneBitMask(self.config.mask))
84 try:
85 image = exposureOrImage.getMaskedImage()
86 except Exception:
87 try:
88 image = exposureOrImage.getImage()
89 except Exception:
90 image = exposureOrImage
91 statType = afwMath.stringToStatisticsProperty(self.config.stat)
92 return afwMath.makeStatistics(image, statType, stats).getValue()
95class CalibCombineConnections(pipeBase.PipelineTaskConnections,
96 dimensions=("instrument", "detector")):
97 inputExps = cT.Input(
98 name="cpInputs",
99 doc="Input pre-processed exposures to combine.",
100 storageClass="Exposure",
101 dimensions=("instrument", "detector", "exposure"),
102 multiple=True,
103 )
104 inputScales = cT.Input(
105 name="cpScales",
106 doc="Input scale factors to use.",
107 storageClass="StructuredDataDict",
108 dimensions=("instrument", ),
109 multiple=False,
110 )
112 outputData = cT.Output(
113 name="cpProposal",
114 doc="Output combined proposed calibration to be validated and certified..",
115 storageClass="ExposureF",
116 dimensions=("instrument", "detector"),
117 isCalibration=True,
118 )
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
123 if config and config.exposureScaling != 'InputList':
124 self.inputs.discard("inputScales")
127# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
128class CalibCombineConfig(pipeBase.PipelineTaskConfig,
129 pipelineConnections=CalibCombineConnections):
130 """Configuration for combining calib exposures.
131 """
132 calibrationType = pexConfig.Field(
133 dtype=str,
134 default="calibration",
135 doc="Name of calibration to be generated.",
136 )
138 exposureScaling = pexConfig.ChoiceField(
139 dtype=str,
140 allowed={
141 "Unity": "Do not scale inputs. Scale factor is 1.0.",
142 "ExposureTime": "Scale inputs by their exposure time.",
143 "DarkTime": "Scale inputs by their dark time.",
144 "MeanStats": "Scale inputs based on their mean values.",
145 "InputList": "Scale inputs based on a list of values.",
146 },
147 default="Unity",
148 doc="Scaling to be applied to each input exposure.",
149 )
150 scalingLevel = pexConfig.ChoiceField(
151 dtype=str,
152 allowed={
153 "DETECTOR": "Scale by detector.",
154 "AMP": "Scale by amplifier.",
155 },
156 default="DETECTOR",
157 doc="Region to scale.",
158 )
159 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
160 dtype=int,
161 default=5,
162 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
163 )
165 doVignette = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Copy vignette polygon to output and censor vignetted pixels?"
169 )
171 mask = pexConfig.ListField(
172 dtype=str,
173 default=["SAT", "DETECTED", "INTRP"],
174 doc="Mask planes to respect",
175 )
176 combine = pexConfig.Field(
177 dtype=str,
178 default='MEANCLIP',
179 doc="Statistic name to use for combination (from lsst.afw.math)",
180 )
181 clip = pexConfig.Field(
182 dtype=float,
183 default=3.0,
184 doc="Clipping threshold for combination",
185 )
186 nIter = pexConfig.Field(
187 dtype=int,
188 default=3,
189 doc="Clipping iterations for combination",
190 )
191 stats = pexConfig.ConfigurableField(
192 target=CalibStatsTask,
193 doc="Background statistics configuration",
194 )
197class CalibCombineTask(pipeBase.PipelineTask,
198 pipeBase.CmdLineTask):
199 """Task to combine calib exposures."""
200 ConfigClass = CalibCombineConfig
201 _DefaultName = 'cpCombine'
203 def __init__(self, **kwargs):
204 super().__init__(**kwargs)
205 self.makeSubtask("stats")
207 def runQuantum(self, butlerQC, inputRefs, outputRefs):
208 inputs = butlerQC.get(inputRefs)
210 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
211 inputs['inputDims'] = dimensions
213 outputs = self.run(**inputs)
214 butlerQC.put(outputs, outputRefs)
216 def run(self, inputExps, inputScales=None, inputDims=None):
217 """Combine calib exposures for a single detector.
219 Parameters
220 ----------
221 inputExps : `list` [`lsst.afw.image.Exposure`]
222 Input list of exposures to combine.
223 inputScales : `dict` [`dict` [`dict` [`float`]]], optional
224 Dictionary of scales, indexed by detector (`int`),
225 amplifier (`int`), and exposure (`int`). Used for
226 'inputList' scaling.
227 inputDims : `list` [`dict`]
228 List of dictionaries of input data dimensions/values.
229 Each list entry should contain:
231 ``"exposure"``
232 exposure id value (`int`)
233 ``"detector"``
234 detector id value (`int`)
236 Returns
237 -------
238 combinedExp : `lsst.afw.image.Exposure`
239 Final combined exposure generated from the inputs.
241 Raises
242 ------
243 RuntimeError
244 Raised if no input data is found. Also raised if
245 config.exposureScaling == InputList, and a necessary scale
246 was not found.
247 """
248 width, height = self.getDimensions(inputExps)
249 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
250 afwImage.Mask.getPlaneBitMask(self.config.mask))
251 numExps = len(inputExps)
252 if numExps < 1:
253 raise RuntimeError("No valid input data")
254 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
255 stats.setCalcErrorFromInputVariance(True)
257 # Check that all inputs either share the same detector (based
258 # on detId), or that no inputs have any detector.
259 detectorList = [exp.getDetector() for exp in inputExps]
260 if None in detectorList:
261 self.log.warn("Not all input detectors defined.")
262 detectorIds = [det.getId() if det is not None else None for det in detectorList]
263 detectorSerials = [det.getId() if det is not None else None for det in detectorList]
264 numDetectorIds = len(set(detectorIds))
265 numDetectorSerials = len(set(detectorSerials))
266 numDetectors = len(set([numDetectorIds, numDetectorSerials]))
267 if numDetectors != 1:
268 raise RuntimeError("Input data contains multiple detectors.")
270 inputDetector = inputExps[0].getDetector()
272 # Create output exposure for combined data.
273 combined = afwImage.MaskedImageF(width, height)
274 combinedExp = afwImage.makeExposure(combined)
276 # Apply scaling:
277 expScales = []
278 if inputDims is None:
279 inputDims = [dict() for i in inputExps]
281 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
282 scale = 1.0
283 if exp is None:
284 self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims)
285 continue
287 if self.config.exposureScaling == "ExposureTime":
288 scale = exp.getInfo().getVisitInfo().getExposureTime()
289 elif self.config.exposureScaling == "DarkTime":
290 scale = exp.getInfo().getVisitInfo().getDarkTime()
291 elif self.config.exposureScaling == "MeanStats":
292 scale = self.stats.run(exp)
293 elif self.config.exposureScaling == "InputList":
294 visitId = dims.get('exposure', None)
295 detectorId = dims.get('detector', None)
296 if visitId is None or detectorId is None:
297 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
298 if detectorId not in inputScales['expScale']:
299 raise RuntimeError(f"Could not identify a scaling for input {index}"
300 f" detector {detectorId}")
302 if self.config.scalingLevel == "DETECTOR":
303 if visitId not in inputScales['expScale'][detectorId]:
304 raise RuntimeError(f"Could not identify a scaling for input {index}"
305 f"detector {detectorId} visit {visitId}")
306 scale = inputScales['expScale'][detectorId][visitId]
307 elif self.config.scalingLevel == 'AMP':
308 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
309 for amp in exp.getDetector()]
310 else:
311 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
312 elif self.config.exposureScaling == 'Unity':
313 scale = 1.0
314 else:
315 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
317 expScales.append(scale)
318 self.log.info("Scaling input %d by %s", index, scale)
319 self.applyScale(exp, scale)
321 self.combine(combined, inputExps, stats)
323 self.interpolateNans(combined)
325 if self.config.doVignette:
326 polygon = inputExps[0].getInfo().getValidPolygon()
327 VignetteExposure(combined, polygon=polygon, doUpdateMask=True,
328 doSetValue=True, vignetteValue=0.0)
330 # Combine headers
331 self.combineHeaders(inputExps, combinedExp,
332 calibType=self.config.calibrationType, scales=expScales)
334 # Set the detector
335 combinedExp.setDetector(inputDetector)
337 # Return
338 return pipeBase.Struct(
339 outputData=combinedExp,
340 )
342 def getDimensions(self, expList):
343 """Get dimensions of the inputs.
345 Parameters
346 ----------
347 expList : `list` [`lsst.afw.image.Exposure`]
348 Exps to check the sizes of.
350 Returns
351 -------
352 width, height : `int`
353 Unique set of input dimensions.
354 """
355 dimList = [exp.getDimensions() for exp in expList if exp is not None]
356 return self.getSize(dimList)
358 def getSize(self, dimList):
359 """Determine a consistent size, given a list of image sizes.
361 Parameters
362 -----------
363 dimList : iterable of `tuple` (`int`, `int`)
364 List of dimensions.
366 Raises
367 ------
368 RuntimeError
369 If input dimensions are inconsistent.
371 Returns
372 --------
373 width, height : `int`
374 Common dimensions.
375 """
376 dim = set((w, h) for w, h in dimList)
377 if len(dim) != 1:
378 raise RuntimeError("Inconsistent dimensions: %s" % dim)
379 return dim.pop()
381 def applyScale(self, exposure, scale=None):
382 """Apply scale to input exposure.
384 This implementation applies a flux scaling: the input exposure is
385 divided by the provided scale.
387 Parameters
388 ----------
389 exposure : `lsst.afw.image.Exposure`
390 Exposure to scale.
391 scale : `float` or `list` [`float`], optional
392 Constant scale to divide the exposure by.
393 """
394 if scale is not None:
395 mi = exposure.getMaskedImage()
396 if isinstance(scale, list):
397 for amp, ampScale in zip(exposure.getDetector(), scale):
398 ampIm = mi[amp.getBBox()]
399 ampIm /= ampScale
400 else:
401 mi /= scale
403 def combine(self, target, expList, stats):
404 """Combine multiple images.
406 Parameters
407 ----------
408 target : `lsst.afw.image.Exposure`
409 Output exposure to construct.
410 expList : `list` [`lsst.afw.image.Exposure`]
411 Input exposures to combine.
412 stats : `lsst.afw.math.StatisticsControl`
413 Control explaining how to combine the input images.
414 """
415 images = [img.getMaskedImage() for img in expList if img is not None]
416 combineType = afwMath.stringToStatisticsProperty(self.config.combine)
417 afwMath.statisticsStack(target, images, combineType, stats)
419 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
420 """Combine input headers to determine the set of common headers,
421 supplemented by calibration inputs.
423 Parameters
424 ----------
425 expList : `list` of `lsst.afw.image.Exposure`
426 Input list of exposures to combine.
427 calib : `lsst.afw.image.Exposure`
428 Output calibration to construct headers for.
429 calibType: `str`, optional
430 OBSTYPE the output should claim.
431 scales: `list` of `float`, optional
432 Scale values applied to each input to record.
434 Returns
435 -------
436 header : `lsst.daf.base.PropertyList`
437 Constructed header.
438 """
439 # Header
440 header = calib.getMetadata()
441 header.set("OBSTYPE", calibType)
443 # Keywords we care about
444 comments = {"TIMESYS": "Time scale for all dates",
445 "DATE-OBS": "Start date of earliest input observation",
446 "MJD-OBS": "[d] Start MJD of earliest input observation",
447 "DATE-END": "End date of oldest input observation",
448 "MJD-END": "[d] End MJD of oldest input observation",
449 "MJD-AVG": "[d] MJD midpoint of all input observations",
450 "DATE-AVG": "Midpoint date of all input observations"}
452 # Creation date
453 now = time.localtime()
454 calibDate = time.strftime("%Y-%m-%d", now)
455 calibTime = time.strftime("%X %Z", now)
456 header.set("CALIB_CREATE_DATE", calibDate)
457 header.set("CALIB_CREATE_TIME", calibTime)
459 # Merge input headers
460 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
461 merged = merge_headers(inputHeaders, mode='drop')
462 for k, v in merged.items():
463 if k not in header:
464 md = expList[0].getMetadata()
465 comment = md.getComment(k) if k in md else None
466 header.set(k, v, comment=comment)
468 # Construct list of visits
469 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
470 for i, visit in enumerate(visitInfoList):
471 if visit is None:
472 continue
473 header.set("CPP_INPUT_%d" % (i,), visit.getExposureId())
474 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
475 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
476 if scales is not None:
477 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
479 # Not yet working: DM-22302
480 # Create an observation group so we can add some standard headers
481 # independent of the form in the input files.
482 # Use try block in case we are dealing with unexpected data headers
483 try:
484 group = ObservationGroup(visitInfoList, pedantic=False)
485 except Exception:
486 self.log.warn("Exception making an obs group for headers. Continuing.")
487 # Fall back to setting a DATE-OBS from the calibDate
488 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
489 comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
490 else:
491 oldest, newest = group.extremes()
492 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
494 for k, v in dateCards.items():
495 header.set(k, v, comment=comments.get(k, None))
497 return header
499 def interpolateNans(self, exp):
500 """Interpolate over NANs in the combined image.
502 NANs can result from masked areas on the CCD. We don't want them getting
503 into our science images, so we replace them with the median of the image.
505 Parameters
506 ----------
507 exp : `lsst.afw.image.Exposure`
508 Exp to check for NaNs.
509 """
510 array = exp.getImage().getArray()
511 bad = np.isnan(array)
513 median = np.median(array[np.logical_not(bad)])
514 count = np.sum(np.logical_not(bad))
515 array[bad] = median
516 if count > 0:
517 self.log.warn("Found %s NAN pixels", count)
520# Create versions of the Connections, Config, and Task that support filter constraints.
521class CalibCombineByFilterConnections(CalibCombineConnections,
522 dimensions=("instrument", "detector", "physical_filter")):
523 inputScales = cT.Input(
524 name="cpFilterScales",
525 doc="Input scale factors to use.",
526 storageClass="StructuredDataDict",
527 dimensions=("instrument", "physical_filter"),
528 multiple=False,
529 )
531 outputData = cT.Output(
532 name="cpFilterProposal",
533 doc="Output combined proposed calibration to be validated and certified.",
534 storageClass="ExposureF",
535 dimensions=("instrument", "detector", "physical_filter"),
536 isCalibration=True,
537 )
539 def __init__(self, *, config=None):
540 super().__init__(config=config)
542 if config and config.exposureScaling != 'InputList':
543 self.inputs.discard("inputScales")
546class CalibCombineByFilterConfig(CalibCombineConfig,
547 pipelineConnections=CalibCombineByFilterConnections):
548 pass
551class CalibCombineByFilterTask(CalibCombineTask):
552 """Task to combine calib exposures."""
553 ConfigClass = CalibCombineByFilterConfig
554 _DefaultName = 'cpFilterCombine'
555 pass
558def VignetteExposure(exposure, polygon=None,
559 doUpdateMask=True, maskPlane='BAD',
560 doSetValue=False, vignetteValue=0.0,
561 log=None):
562 """Apply vignetted polygon to image pixels.
564 Parameters
565 ----------
566 exposure : `lsst.afw.image.Exposure`
567 Image to be updated.
568 doUpdateMask : `bool`, optional
569 Update the exposure mask for vignetted area?
570 maskPlane : `str`, optional,
571 Mask plane to assign.
572 doSetValue : `bool`, optional
573 Set image value for vignetted area?
574 vignetteValue : `float`, optional
575 Value to assign.
576 log : `lsst.log.Log`, optional
577 Log to write to.
579 Raises
580 ------
581 RuntimeError
582 Raised if no valid polygon exists.
583 """
584 polygon = polygon if polygon else exposure.getInfo().getValidPolygon()
585 if not polygon:
586 raise RuntimeError("Could not find valid polygon!")
587 log = log if log else Log.getLogger(__name__.partition(".")[2])
589 fullyIlluminated = True
590 for corner in exposure.getBBox().getCorners():
591 if not polygon.contains(Point2D(corner)):
592 fullyIlluminated = False
594 log.info("Exposure is fully illuminated? %s", fullyIlluminated)
596 if not fullyIlluminated:
597 # Scan pixels.
598 mask = exposure.getMask()
599 numPixels = mask.getBBox().getArea()
601 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
602 np.arange(0, mask.getHeight(), dtype=int))
604 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in
605 zip(xx.reshape(numPixels), yy.reshape(numPixels))])
606 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
608 if doUpdateMask:
609 bitMask = mask.getPlaneBitMask(maskPlane)
610 maskArray = mask.getArray()
611 maskArray[vignMask] |= bitMask
612 if doSetValue:
613 imageArray = exposure.getImage().getArray()
614 imageArray[vignMask] = vignetteValue
615 log.info("Exposure contains %d vignetted pixels.",
616 np.count_nonzero(vignMask))