Coverage for python/lsst/cp/pipe/cpCombine.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_pipe.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import time
24import lsst.pex.config as pexConfig
25import lsst.pipe.base as pipeBase
26import lsst.pipe.base.connectionTypes as cT
27import lsst.afw.math as afwMath
28import lsst.afw.image as afwImage
30from lsst.geom import Point2D
31from lsst.log import Log
32from astro_metadata_translator import merge_headers, ObservationGroup
33from astro_metadata_translator.serialize import dates_to_fits
36# CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
37class CalibStatsConfig(pexConfig.Config):
38 """Parameters controlling the measurement of background statistics.
39 """
40 stat = pexConfig.Field(
41 dtype=str,
42 default='MEANCLIP',
43 doc="Statistic name to use to estimate background (from lsst.afw.math)",
44 )
45 clip = pexConfig.Field(
46 dtype=float,
47 default=3.0,
48 doc="Clipping threshold for background",
49 )
50 nIter = pexConfig.Field(
51 dtype=int,
52 default=3,
53 doc="Clipping iterations for background",
54 )
55 mask = pexConfig.ListField(
56 dtype=str,
57 default=["DETECTED", "BAD", "NO_DATA"],
58 doc="Mask planes to reject",
59 )
62class CalibStatsTask(pipeBase.Task):
63 """Measure statistics on the background
65 This can be useful for scaling the background, e.g., for flats and fringe frames.
66 """
67 ConfigClass = CalibStatsConfig
69 def run(self, exposureOrImage):
70 """Measure a particular statistic on an image (of some sort).
72 Parameters
73 ----------
74 exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75 Exposure or image to calculate statistics on.
77 Returns
78 -------
79 results : float
80 Resulting statistic value.
81 """
82 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83 afwImage.Mask.getPlaneBitMask(self.config.mask))
84 try:
85 image = exposureOrImage.getMaskedImage()
86 except Exception:
87 try:
88 image = exposureOrImage.getImage()
89 except Exception:
90 image = exposureOrImage
91 statType = afwMath.stringToStatisticsProperty(self.config.stat)
92 return afwMath.makeStatistics(image, statType, stats).getValue()
95class CalibCombineConnections(pipeBase.PipelineTaskConnections,
96 dimensions=("instrument", "detector")):
97 inputExps = cT.Input(
98 name="cpInputs",
99 doc="Input pre-processed exposures to combine.",
100 storageClass="Exposure",
101 dimensions=("instrument", "detector", "exposure"),
102 multiple=True,
103 )
104 inputScales = cT.Input(
105 name="cpScales",
106 doc="Input scale factors to use.",
107 storageClass="StructuredDataDict",
108 dimensions=("instrument", ),
109 multiple=False,
110 )
112 outputData = cT.Output(
113 name="cpProposal",
114 doc="Output combined proposed calibration to be validated and certified..",
115 storageClass="ExposureF",
116 dimensions=("instrument", "detector"),
117 isCalibration=True,
118 )
120 def __init__(self, *, config=None):
121 super().__init__(config=config)
123 if config and config.exposureScaling != 'InputList':
124 self.inputs.discard("inputScales")
127# CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
128class CalibCombineConfig(pipeBase.PipelineTaskConfig,
129 pipelineConnections=CalibCombineConnections):
130 """Configuration for combining calib exposures.
131 """
132 calibrationType = pexConfig.Field(
133 dtype=str,
134 default="calibration",
135 doc="Name of calibration to be generated.",
136 )
138 exposureScaling = pexConfig.ChoiceField(
139 dtype=str,
140 allowed={
141 "Unity": "Do not scale inputs. Scale factor is 1.0.",
142 "ExposureTime": "Scale inputs by their exposure time.",
143 "DarkTime": "Scale inputs by their dark time.",
144 "MeanStats": "Scale inputs based on their mean values.",
145 "InputList": "Scale inputs based on a list of values.",
146 },
147 default="Unity",
148 doc="Scaling to be applied to each input exposure.",
149 )
150 scalingLevel = pexConfig.ChoiceField(
151 dtype=str,
152 allowed={
153 "DETECTOR": "Scale by detector.",
154 "AMP": "Scale by amplifier.",
155 },
156 default="DETECTOR",
157 doc="Region to scale.",
158 )
159 maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
160 dtype=int,
161 default=5,
162 doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
163 )
165 doVignette = pexConfig.Field(
166 dtype=bool,
167 default=False,
168 doc="Copy vignette polygon to output and censor vignetted pixels?"
169 )
171 mask = pexConfig.ListField(
172 dtype=str,
173 default=["SAT", "DETECTED", "INTRP"],
174 doc="Mask planes to respect",
175 )
176 combine = pexConfig.Field(
177 dtype=str,
178 default='MEANCLIP',
179 doc="Statistic name to use for combination (from lsst.afw.math)",
180 )
181 clip = pexConfig.Field(
182 dtype=float,
183 default=3.0,
184 doc="Clipping threshold for combination",
185 )
186 nIter = pexConfig.Field(
187 dtype=int,
188 default=3,
189 doc="Clipping iterations for combination",
190 )
191 stats = pexConfig.ConfigurableField(
192 target=CalibStatsTask,
193 doc="Background statistics configuration",
194 )
197class CalibCombineTask(pipeBase.PipelineTask,
198 pipeBase.CmdLineTask):
199 """Task to combine calib exposures."""
200 ConfigClass = CalibCombineConfig
201 _DefaultName = 'cpCombine'
203 def __init__(self, **kwargs):
204 super().__init__(**kwargs)
205 self.makeSubtask("stats")
207 def runQuantum(self, butlerQC, inputRefs, outputRefs):
208 inputs = butlerQC.get(inputRefs)
210 dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
211 inputs['inputDims'] = dimensions
213 outputs = self.run(**inputs)
214 butlerQC.put(outputs, outputRefs)
216 def run(self, inputExps, inputScales=None, inputDims=None):
217 """Combine calib exposures for a single detector.
219 Parameters
220 ----------
221 inputExps : `list` [`lsst.afw.image.Exposure`]
222 Input list of exposures to combine.
223 inputScales : `dict` [`dict` [`dict` [`float`]]], optional
224 Dictionary of scales, indexed by detector (`int`),
225 amplifier (`int`), and exposure (`int`). Used for
226 'inputList' scaling.
227 inputDims : `list` [`dict`]
228 List of dictionaries of input data dimensions/values.
229 Each list entry should contain:
231 ``"exposure"``
232 exposure id value (`int`)
233 ``"detector"``
234 detector id value (`int`)
236 Returns
237 -------
238 combinedExp : `lsst.afw.image.Exposure`
239 Final combined exposure generated from the inputs.
241 Raises
242 ------
243 RuntimeError
244 Raised if no input data is found. Also raised if
245 config.exposureScaling == InputList, and a necessary scale
246 was not found.
247 """
248 width, height = self.getDimensions(inputExps)
249 stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
250 afwImage.Mask.getPlaneBitMask(self.config.mask))
251 numExps = len(inputExps)
252 if numExps < 1:
253 raise RuntimeError("No valid input data")
254 if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
255 stats.setCalcErrorFromInputVariance(True)
257 # Create output exposure for combined data.
258 combined = afwImage.MaskedImageF(width, height)
259 combinedExp = afwImage.makeExposure(combined)
261 # Apply scaling:
262 expScales = []
263 if inputDims is None:
264 inputDims = [dict() for i in inputExps]
266 for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
267 scale = 1.0
268 if exp is None:
269 self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims)
270 continue
272 if self.config.exposureScaling == "ExposureTime":
273 scale = exp.getInfo().getVisitInfo().getExposureTime()
274 elif self.config.exposureScaling == "DarkTime":
275 scale = exp.getInfo().getVisitInfo().getDarkTime()
276 elif self.config.exposureScaling == "MeanStats":
277 scale = self.stats.run(exp)
278 elif self.config.exposureScaling == "InputList":
279 visitId = dims.get('exposure', None)
280 detectorId = dims.get('detector', None)
281 if visitId is None or detectorId is None:
282 raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
283 if detectorId not in inputScales['expScale']:
284 raise RuntimeError(f"Could not identify a scaling for input {index}"
285 f" detector {detectorId}")
287 if self.config.scalingLevel == "DETECTOR":
288 if visitId not in inputScales['expScale'][detectorId]:
289 raise RuntimeError(f"Could not identify a scaling for input {index}"
290 f"detector {detectorId} visit {visitId}")
291 scale = inputScales['expScale'][detectorId][visitId]
292 elif self.config.scalingLevel == 'AMP':
293 scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
294 for amp in exp.getDetector()]
295 else:
296 raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
297 elif self.config.exposureScaling == 'Unity':
298 scale = 1.0
299 else:
300 raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
302 expScales.append(scale)
303 self.log.info("Scaling input %d by %s", index, scale)
304 self.applyScale(exp, scale)
306 self.combine(combined, inputExps, stats)
308 self.interpolateNans(combined)
310 if self.config.doVignette:
311 polygon = inputExps[0].getInfo().getValidPolygon()
312 VignetteExposure(combined, polygon=polygon, doUpdateMask=True,
313 doSetValue=True, vignetteValue=0.0)
315 # Combine headers
316 self.combineHeaders(inputExps, combinedExp,
317 calibType=self.config.calibrationType, scales=expScales)
319 # Return
320 return pipeBase.Struct(
321 outputData=combinedExp,
322 )
324 def getDimensions(self, expList):
325 """Get dimensions of the inputs.
327 Parameters
328 ----------
329 expList : `list` [`lsst.afw.image.Exposure`]
330 Exps to check the sizes of.
332 Returns
333 -------
334 width, height : `int`
335 Unique set of input dimensions.
336 """
337 dimList = [exp.getDimensions() for exp in expList if exp is not None]
338 return self.getSize(dimList)
340 def getSize(self, dimList):
341 """Determine a consistent size, given a list of image sizes.
343 Parameters
344 -----------
345 dimList : iterable of `tuple` (`int`, `int`)
346 List of dimensions.
348 Raises
349 ------
350 RuntimeError
351 If input dimensions are inconsistent.
353 Returns
354 --------
355 width, height : `int`
356 Common dimensions.
357 """
358 dim = set((w, h) for w, h in dimList)
359 if len(dim) != 1:
360 raise RuntimeError("Inconsistent dimensions: %s" % dim)
361 return dim.pop()
363 def applyScale(self, exposure, scale=None):
364 """Apply scale to input exposure.
366 This implementation applies a flux scaling: the input exposure is
367 divided by the provided scale.
369 Parameters
370 ----------
371 exposure : `lsst.afw.image.Exposure`
372 Exposure to scale.
373 scale : `float` or `list` [`float`], optional
374 Constant scale to divide the exposure by.
375 """
376 if scale is not None:
377 mi = exposure.getMaskedImage()
378 if isinstance(scale, list):
379 for amp, ampScale in zip(exposure.getDetector(), scale):
380 ampIm = mi[amp.getBBox()]
381 ampIm /= ampScale
382 else:
383 mi /= scale
385 def combine(self, target, expList, stats):
386 """Combine multiple images.
388 Parameters
389 ----------
390 target : `lsst.afw.image.Exposure`
391 Output exposure to construct.
392 expList : `list` [`lsst.afw.image.Exposure`]
393 Input exposures to combine.
394 stats : `lsst.afw.math.StatisticsControl`
395 Control explaining how to combine the input images.
396 """
397 images = [img.getMaskedImage() for img in expList if img is not None]
398 combineType = afwMath.stringToStatisticsProperty(self.config.combine)
399 afwMath.statisticsStack(target, images, combineType, stats)
401 def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
402 """Combine input headers to determine the set of common headers,
403 supplemented by calibration inputs.
405 Parameters
406 ----------
407 expList : `list` of `lsst.afw.image.Exposure`
408 Input list of exposures to combine.
409 calib : `lsst.afw.image.Exposure`
410 Output calibration to construct headers for.
411 calibType: `str`, optional
412 OBSTYPE the output should claim.
413 scales: `list` of `float`, optional
414 Scale values applied to each input to record.
416 Returns
417 -------
418 header : `lsst.daf.base.PropertyList`
419 Constructed header.
420 """
421 # Header
422 header = calib.getMetadata()
423 header.set("OBSTYPE", calibType)
425 # Keywords we care about
426 comments = {"TIMESYS": "Time scale for all dates",
427 "DATE-OBS": "Start date of earliest input observation",
428 "MJD-OBS": "[d] Start MJD of earliest input observation",
429 "DATE-END": "End date of oldest input observation",
430 "MJD-END": "[d] End MJD of oldest input observation",
431 "MJD-AVG": "[d] MJD midpoint of all input observations",
432 "DATE-AVG": "Midpoint date of all input observations"}
434 # Creation date
435 now = time.localtime()
436 calibDate = time.strftime("%Y-%m-%d", now)
437 calibTime = time.strftime("%X %Z", now)
438 header.set("CALIB_CREATE_DATE", calibDate)
439 header.set("CALIB_CREATE_TIME", calibTime)
441 # Merge input headers
442 inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
443 merged = merge_headers(inputHeaders, mode='drop')
444 for k, v in merged.items():
445 if k not in header:
446 md = expList[0].getMetadata()
447 comment = md.getComment(k) if k in md else None
448 header.set(k, v, comment=comment)
450 # Construct list of visits
451 visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
452 for i, visit in enumerate(visitInfoList):
453 if visit is None:
454 continue
455 header.set("CPP_INPUT_%d" % (i,), visit.getExposureId())
456 header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
457 header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
458 if scales is not None:
459 header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
461 # Not yet working: DM-22302
462 # Create an observation group so we can add some standard headers
463 # independent of the form in the input files.
464 # Use try block in case we are dealing with unexpected data headers
465 try:
466 group = ObservationGroup(visitInfoList, pedantic=False)
467 except Exception:
468 self.log.warn("Exception making an obs group for headers. Continuing.")
469 # Fall back to setting a DATE-OBS from the calibDate
470 dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
471 comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
472 else:
473 oldest, newest = group.extremes()
474 dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
476 for k, v in dateCards.items():
477 header.set(k, v, comment=comments.get(k, None))
479 return header
481 def interpolateNans(self, exp):
482 """Interpolate over NANs in the combined image.
484 NANs can result from masked areas on the CCD. We don't want them getting
485 into our science images, so we replace them with the median of the image.
487 Parameters
488 ----------
489 exp : `lsst.afw.image.Exposure`
490 Exp to check for NaNs.
491 """
492 array = exp.getImage().getArray()
493 bad = np.isnan(array)
495 median = np.median(array[np.logical_not(bad)])
496 count = np.sum(np.logical_not(bad))
497 array[bad] = median
498 if count > 0:
499 self.log.warn("Found %s NAN pixels", count)
502# Create versions of the Connections, Config, and Task that support filter constraints.
503class CalibCombineByFilterConnections(CalibCombineConnections,
504 dimensions=("instrument", "detector", "physical_filter")):
505 inputScales = cT.Input(
506 name="cpFilterScales",
507 doc="Input scale factors to use.",
508 storageClass="StructuredDataDict",
509 dimensions=("instrument", "physical_filter"),
510 multiple=False,
511 )
513 outputData = cT.Output(
514 name="cpFilterProposal",
515 doc="Output combined proposed calibration to be validated and certified.",
516 storageClass="ExposureF",
517 dimensions=("instrument", "detector", "physical_filter"),
518 isCalibration=True,
519 )
521 def __init__(self, *, config=None):
522 super().__init__(config=config)
524 if config and config.exposureScaling != 'InputList':
525 self.inputs.discard("inputScales")
528class CalibCombineByFilterConfig(CalibCombineConfig,
529 pipelineConnections=CalibCombineByFilterConnections):
530 pass
533class CalibCombineByFilterTask(CalibCombineTask):
534 """Task to combine calib exposures."""
535 ConfigClass = CalibCombineByFilterConfig
536 _DefaultName = 'cpFilterCombine'
537 pass
540def VignetteExposure(exposure, polygon=None,
541 doUpdateMask=True, maskPlane='BAD',
542 doSetValue=False, vignetteValue=0.0,
543 log=None):
544 """Apply vignetted polygon to image pixels.
546 Parameters
547 ----------
548 exposure : `lsst.afw.image.Exposure`
549 Image to be updated.
550 doUpdateMask : `bool`, optional
551 Update the exposure mask for vignetted area?
552 maskPlane : `str`, optional,
553 Mask plane to assign.
554 doSetValue : `bool`, optional
555 Set image value for vignetted area?
556 vignetteValue : `float`, optional
557 Value to assign.
558 log : `lsst.log.Log`, optional
559 Log to write to.
561 Raises
562 ------
563 RuntimeError
564 Raised if no valid polygon exists.
565 """
566 polygon = polygon if polygon else exposure.getInfo().getValidPolygon()
567 if not polygon:
568 raise RuntimeError("Could not find valid polygon!")
569 log = log if log else Log.getLogger(__name__.partition(".")[2])
571 fullyIlluminated = True
572 for corner in exposure.getBBox().getCorners():
573 if not polygon.contains(Point2D(corner)):
574 fullyIlluminated = False
576 log.info("Exposure is fully illuminated? %s", fullyIlluminated)
578 if not fullyIlluminated:
579 # Scan pixels.
580 mask = exposure.getMask()
581 numPixels = mask.getBBox().getArea()
583 xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
584 np.arange(0, mask.getHeight(), dtype=int))
586 vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in
587 zip(xx.reshape(numPixels), yy.reshape(numPixels))])
588 vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
590 if doUpdateMask:
591 bitMask = mask.getPlaneBitMask(maskPlane)
592 maskArray = mask.getArray()
593 maskArray[vignMask] |= bitMask
594 if doSetValue:
595 imageArray = exposure.getImage().getArray()
596 imageArray[vignMask] = vignetteValue
597 log.info("Exposure contains %d vignetted pixels.",
598 np.count_nonzero(vignMask))