lsst.cp.pipe  20.0.0-14-gbeb4550+636d787cb5
cpCombine.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import numpy as np
22 import time
23 
24 import lsst.pex.config as pexConfig
25 import lsst.pipe.base as pipeBase
27 import lsst.afw.math as afwMath
28 import lsst.afw.image as afwImage
29 
30 from lsst.geom import Point2D
31 from lsst.log import Log
32 from astro_metadata_translator import merge_headers, ObservationGroup
33 from astro_metadata_translator.serialize import dates_to_fits
34 
35 
36 # CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
37 class CalibStatsConfig(pexConfig.Config):
38  """Parameters controlling the measurement of background statistics.
39  """
40  stat = pexConfig.Field(
41  dtype=int,
42  default=int(afwMath.MEANCLIP),
43  doc="Statistic to use to estimate background (from lsst.afw.math)",
44  )
45  clip = pexConfig.Field(
46  dtype=float,
47  default=3.0,
48  doc="Clipping threshold for background",
49  )
50  nIter = pexConfig.Field(
51  dtype=int,
52  default=3,
53  doc="Clipping iterations for background",
54  )
55  mask = pexConfig.ListField(
56  dtype=str,
57  default=["DETECTED", "BAD", "NO_DATA"],
58  doc="Mask planes to reject",
59  )
60 
61 
62 class CalibStatsTask(pipeBase.Task):
63  """Measure statistics on the background
64 
65  This can be useful for scaling the background, e.g., for flats and fringe frames.
66  """
67  ConfigClass = CalibStatsConfig
68 
69  def run(self, exposureOrImage):
70  """Measure a particular statistic on an image (of some sort).
71 
72  Parameters
73  ----------
74  exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75  Exposure or image to calculate statistics on.
76 
77  Returns
78  -------
79  results : float
80  Resulting statistic value.
81  """
82  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83  afwImage.Mask.getPlaneBitMask(self.config.mask))
84  try:
85  image = exposureOrImage.getMaskedImage()
86  except Exception:
87  try:
88  image = exposureOrImage.getImage()
89  except Exception:
90  image = exposureOrImage
91 
92  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
93 
94 
95 class CalibCombineConnections(pipeBase.PipelineTaskConnections,
96  dimensions=("instrument", "detector")):
97  inputExps = cT.Input(
98  name="cpInputs",
99  doc="Input pre-processed exposures to combine.",
100  storageClass="Exposure",
101  dimensions=("instrument", "detector", "exposure"),
102  multiple=True,
103  )
104  inputScales = cT.Input(
105  name="cpScales",
106  doc="Input scale factors to use.",
107  storageClass="StructuredDataDict",
108  dimensions=("instrument", ),
109  multiple=False,
110  )
111 
112  outputData = cT.Output(
113  name="cpProposal",
114  doc="Output combined proposed calibration.",
115  storageClass="ExposureF",
116  dimensions=("instrument", "detector"),
117  )
118 
119  def __init__(self, *, config=None):
120  super().__init__(config=config)
121 
122  if config and config.exposureScaling != 'InputList':
123  self.inputs.discard("inputScales")
124 
125  if config and len(config.calibrationDimensions) != 0:
126  newDimensions = tuple(config.calibrationDimensions)
127  newOutputData = cT.Output(
128  name=self.outputData.name,
129  doc=self.outputData.doc,
130  storageClass=self.outputData.storageClass,
131  dimensions=self.allConnections['outputData'].dimensions + newDimensions
132  )
133  self.dimensions.update(config.calibrationDimensions)
134  self.outputData = newOutputData
135 
136  if config.exposureScaling == 'InputList':
137  newInputScales = cT.PrerequisiteInput(
138  name=self.inputScales.name,
139  doc=self.inputScales.doc,
140  storageClass=self.inputScales.storageClass,
141  dimensions=self.allConnections['inputScales'].dimensions + newDimensions
142  )
143  self.dimensions.update(config.calibrationDimensions)
144  self.inputScales = newInputScales
145 
146 
147 # CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
148 class CalibCombineConfig(pipeBase.PipelineTaskConfig,
149  pipelineConnections=CalibCombineConnections):
150  """Configuration for combining calib exposures.
151  """
152  calibrationType = pexConfig.Field(
153  dtype=str,
154  default="calibration",
155  doc="Name of calibration to be generated.",
156  )
157  calibrationDimensions = pexConfig.ListField(
158  dtype=str,
159  default=[],
160  doc="List of updated dimensions to append to output."
161  )
162 
163  exposureScaling = pexConfig.ChoiceField(
164  dtype=str,
165  allowed={
166  "None": "No scaling used.",
167  "ExposureTime": "Scale inputs by their exposure time.",
168  "DarkTime": "Scale inputs by their dark time.",
169  "MeanStats": "Scale inputs based on their mean values.",
170  "InputList": "Scale inputs based on a list of values.",
171  },
172  default=None,
173  doc="Scaling to be applied to each input exposure.",
174  )
175  scalingLevel = pexConfig.ChoiceField(
176  dtype=str,
177  allowed={
178  "DETECTOR": "Scale by detector.",
179  "AMP": "Scale by amplifier.",
180  },
181  default="DETECTOR",
182  doc="Region to scale.",
183  )
184  maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
185  dtype=int,
186  default=5,
187  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
188  )
189 
190  doVignette = pexConfig.Field(
191  dtype=bool,
192  default=False,
193  doc="Copy vignette polygon to output and censor vignetted pixels?"
194  )
195 
196  mask = pexConfig.ListField(
197  dtype=str,
198  default=["SAT", "DETECTED", "INTRP"],
199  doc="Mask planes to respect",
200  )
201  combine = pexConfig.Field(
202  dtype=int,
203  default=int(afwMath.MEANCLIP),
204  doc="Statistic to use for combination (from lsst.afw.math)",
205  )
206  clip = pexConfig.Field(
207  dtype=float,
208  default=3.0,
209  doc="Clipping threshold for combination",
210  )
211  nIter = pexConfig.Field(
212  dtype=int,
213  default=3,
214  doc="Clipping iterations for combination",
215  )
216  stats = pexConfig.ConfigurableField(
217  target=CalibStatsTask,
218  doc="Background statistics configuration",
219  )
220 
221 
222 class CalibCombineTask(pipeBase.PipelineTask,
223  pipeBase.CmdLineTask):
224  """Task to combine calib exposures."""
225  ConfigClass = CalibCombineConfig
226  _DefaultName = 'cpCombine'
227 
228  def __init__(self, **kwargs):
229  super().__init__(**kwargs)
230  self.makeSubtask("stats")
231 
232  def runQuantum(self, butlerQC, inputRefs, outputRefs):
233  inputs = butlerQC.get(inputRefs)
234 
235  dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
236  inputs['inputDims'] = dimensions
237 
238  outputs = self.run(**inputs)
239  butlerQC.put(outputs, outputRefs)
240 
241  def run(self, inputExps, inputScales=None, inputDims=None):
242  """Combine calib exposures for a single detector.
243 
244  Parameters
245  ----------
246  inputExps : `list` [`lsst.afw.image.Exposure`]
247  Input list of exposures to combine.
248  inputScales : `dict` [`dict` [`dict` [`float`]]], optional
249  Dictionary of scales, indexed by detector (`int`),
250  amplifier (`int`), and exposure (`int`). Used for
251  'inputList' scaling.
252  inputDims : `list` [`dict`]
253  List of dictionaries of input data dimensions/values.
254  Each list entry should contain:
255 
256  ``"exposure"``
257  exposure id value (`int`)
258  ``"detector"``
259  detector id value (`int`)
260 
261  Returns
262  -------
263  combinedExp : `lsst.afw.image.Exposure`
264  Final combined exposure generated from the inputs.
265 
266  Raises
267  ------
268  RuntimeError
269  Raised if no input data is found. Also raised if
270  config.exposureScaling == InputList, and a necessary scale
271  was not found.
272  """
273  width, height = self.getDimensions(inputExps)
274  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
275  afwImage.Mask.getPlaneBitMask(self.config.mask))
276  numExps = len(inputExps)
277  if numExps < 1:
278  raise RuntimeError("No valid input data")
279  if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
280  stats.setCalcErrorFromInputVariance(True)
281 
282  # Create output exposure for combined data.
283  combined = afwImage.MaskedImageF(width, height)
284  combinedExp = afwImage.makeExposure(combined)
285 
286  # Apply scaling:
287  expScales = []
288  if inputDims is None:
289  inputDims = [dict() for i in inputExps]
290 
291  for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
292  scale = 1.0
293  if exp is None:
294  self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims)
295  continue
296 
297  if self.config.exposureScaling == "ExposureTime":
298  scale = exp.getInfo().getVisitInfo().getExposureTime()
299  elif self.config.exposureScaling == "DarkTime":
300  scale = exp.getInfo().getVisitInfo().getDarkTime()
301  elif self.config.exposureScaling == "MeanStats":
302  scale = self.stats.run(exp)
303  elif self.config.exposureScaling == "InputList":
304  visitId = dims.get('exposure', None)
305  detectorId = dims.get('detector', None)
306  if visitId is None or detectorId is None:
307  raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
308  if detectorId not in inputScales['expScale']:
309  raise RuntimeError(f"Could not identify a scaling for input {index}"
310  f" detector {detectorId}")
311 
312  if self.config.scalingLevel == "DETECTOR":
313  if visitId not in inputScales['expScale'][detectorId]:
314  raise RuntimeError(f"Could not identify a scaling for input {index}"
315  f"detector {detectorId} visit {visitId}")
316  scale = inputScales['expScale'][detectorId][visitId]
317  elif self.config.scalingLevel == 'AMP':
318  scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
319  for amp in exp.getDetector()]
320  else:
321  raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
322  elif self.config.exposureScaling == 'None':
323  scale = 1.0
324  else:
325  raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
326 
327  expScales.append(scale)
328  self.log.info("Scaling input %d by %s", index, scale)
329  self.applyScale(exp, scale)
330 
331  self.combine(combined, inputExps, stats)
332 
333  self.interpolateNans(combined)
334 
335  if self.config.doVignette:
336  polygon = inputExps[0].getInfo().getValidPolygon()
337  VignetteExposure(combined, polygon=polygon, doUpdateMask=True,
338  doSetValue=True, vignetteValue=0.0)
339 
340  # Combine headers
341  self.combineHeaders(inputExps, combinedExp,
342  calibType=self.config.calibrationType, scales=expScales)
343 
344  # Return
345  return pipeBase.Struct(
346  outputData=combinedExp,
347  )
348 
349  def getDimensions(self, expList):
350  """Get dimensions of the inputs.
351 
352  Parameters
353  ----------
354  expList : `list` [`lsst.afw.image.Exposure`]
355  Exps to check the sizes of.
356 
357  Returns
358  -------
359  width, height : `int`
360  Unique set of input dimensions.
361  """
362  dimList = [exp.getDimensions() for exp in expList if exp is not None]
363  return self.getSize(dimList)
364 
365  def getSize(self, dimList):
366  """Determine a consistent size, given a list of image sizes.
367 
368  Parameters
369  -----------
370  dimList : iterable of `tuple` (`int`, `int`)
371  List of dimensions.
372 
373  Raises
374  ------
375  RuntimeError
376  If input dimensions are inconsistent.
377 
378  Returns
379  --------
380  width, height : `int`
381  Common dimensions.
382  """
383  dim = set((w, h) for w, h in dimList)
384  if len(dim) != 1:
385  raise RuntimeError("Inconsistent dimensions: %s" % dim)
386  return dim.pop()
387 
388  def applyScale(self, exposure, scale=None):
389  """Apply scale to input exposure.
390 
391  This implementation applies a flux scaling: the input exposure is
392  divided by the provided scale.
393 
394  Parameters
395  ----------
396  exposure : `lsst.afw.image.Exposure`
397  Exposure to scale.
398  scale : `float` or `list` [`float`], optional
399  Constant scale to divide the exposure by.
400  """
401  if scale is not None:
402  mi = exposure.getMaskedImage()
403  if isinstance(scale, list):
404  for amp, ampScale in zip(exposure.getDetector(), scale):
405  ampIm = mi[amp.getBBox()]
406  ampIm /= ampScale
407  else:
408  mi /= scale
409 
410  def combine(self, target, expList, stats):
411  """Combine multiple images.
412 
413  Parameters
414  ----------
415  target : `lsst.afw.image.Exposure`
416  Output exposure to construct.
417  expList : `list` [`lsst.afw.image.Exposure`]
418  Input exposures to combine.
419  stats : `lsst.afw.math.StatisticsControl`
420  Control explaining how to combine the input images.
421  """
422  images = [img.getMaskedImage() for img in expList if img is not None]
423  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
424 
425  def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
426  """Combine input headers to determine the set of common headers,
427  supplemented by calibration inputs.
428 
429  Parameters
430  ----------
431  expList : `list` of `lsst.afw.image.Exposure`
432  Input list of exposures to combine.
433  calib : `lsst.afw.image.Exposure`
434  Output calibration to construct headers for.
435  calibType: `str`, optional
436  OBSTYPE the output should claim.
437  scales: `list` of `float`, optional
438  Scale values applied to each input to record.
439 
440  Returns
441  -------
442  header : `lsst.daf.base.PropertyList`
443  Constructed header.
444  """
445  # Header
446  header = calib.getMetadata()
447  header.set("OBSTYPE", calibType)
448 
449  # Keywords we care about
450  comments = {"TIMESYS": "Time scale for all dates",
451  "DATE-OBS": "Start date of earliest input observation",
452  "MJD-OBS": "[d] Start MJD of earliest input observation",
453  "DATE-END": "End date of oldest input observation",
454  "MJD-END": "[d] End MJD of oldest input observation",
455  "MJD-AVG": "[d] MJD midpoint of all input observations",
456  "DATE-AVG": "Midpoint date of all input observations"}
457 
458  # Creation date
459  now = time.localtime()
460  calibDate = time.strftime("%Y-%m-%d", now)
461  calibTime = time.strftime("%X %Z", now)
462  header.set("CALIB_CREATE_DATE", calibDate)
463  header.set("CALIB_CREATE_TIME", calibTime)
464 
465  # Merge input headers
466  inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
467  merged = merge_headers(inputHeaders, mode='drop')
468  for k, v in merged.items():
469  if k not in header:
470  md = expList[0].getMetadata()
471  comment = md.getComment(k) if k in md else None
472  header.set(k, v, comment=comment)
473 
474  # Construct list of visits
475  visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
476  for i, visit in enumerate(visitInfoList):
477  if visit is None:
478  continue
479  header.set("CPP_INPUT_%d" % (i,), visit.getExposureId())
480  header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
481  header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
482  if scales is not None:
483  header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
484 
485  # Not yet working: DM-22302
486  # Create an observation group so we can add some standard headers
487  # independent of the form in the input files.
488  # Use try block in case we are dealing with unexpected data headers
489  try:
490  group = ObservationGroup(visitInfoList, pedantic=False)
491  except Exception:
492  self.log.warn("Exception making an obs group for headers. Continuing.")
493  # Fall back to setting a DATE-OBS from the calibDate
494  dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
495  comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
496  else:
497  oldest, newest = group.extremes()
498  dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
499 
500  for k, v in dateCards.items():
501  header.set(k, v, comment=comments.get(k, None))
502 
503  return header
504 
505  def interpolateNans(self, exp):
506  """Interpolate over NANs in the combined image.
507 
508  NANs can result from masked areas on the CCD. We don't want them getting
509  into our science images, so we replace them with the median of the image.
510 
511  Parameters
512  ----------
513  exp : `lsst.afw.image.Exposure`
514  Exp to check for NaNs.
515  """
516  array = exp.getImage().getArray()
517  bad = np.isnan(array)
518 
519  median = np.median(array[np.logical_not(bad)])
520  count = np.sum(np.logical_not(bad))
521  array[bad] = median
522  if count > 0:
523  self.log.warn("Found %s NAN pixels", count)
524 
525 
526 def VignetteExposure(exposure, polygon=None,
527  doUpdateMask=True, maskPlane='BAD',
528  doSetValue=False, vignetteValue=0.0,
529  log=None):
530  """Apply vignetted polygon to image pixels.
531 
532  Parameters
533  ----------
534  exposure : `lsst.afw.image.Exposure`
535  Image to be updated.
536  doUpdateMask : `bool`, optional
537  Update the exposure mask for vignetted area?
538  maskPlane : `str`, optional,
539  Mask plane to assign.
540  doSetValue : `bool`, optional
541  Set image value for vignetted area?
542  vignetteValue : `float`, optional
543  Value to assign.
544  log : `lsst.log.Log`, optional
545  Log to write to.
546 
547  Raises
548  ------
549  RuntimeError
550  Raised if no valid polygon exists.
551  """
552  polygon = polygon if polygon else exposure.getInfo().getValidPolygon()
553  if not polygon:
554  raise RuntimeError("Could not find valid polygon!")
555  log = log if log else Log.getLogger(__name__.partition(".")[2])
556 
557  fullyIlluminated = True
558  for corner in exposure.getBBox().getCorners():
559  if not polygon.contains(Point2D(corner)):
560  fullyIlluminated = False
561 
562  log.info("Exposure is fully illuminated? %s", fullyIlluminated)
563 
564  if not fullyIlluminated:
565  # Scan pixels.
566  mask = exposure.getMask()
567  numPixels = mask.getBBox().getArea()
568 
569  xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
570  np.arange(0, mask.getHeight(), dtype=int))
571 
572  vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in
573  zip(xx.reshape(numPixels), yy.reshape(numPixels))])
574  vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
575 
576  if doUpdateMask:
577  bitMask = mask.getPlaneBitMask(maskPlane)
578  maskArray = mask.getArray()
579  maskArray[vignMask] |= bitMask
580  if doSetValue:
581  imageArray = exposure.getImage().getArray()
582  imageArray[vignMask] = vignetteValue
583  log.info("Exposure contains %d vignetted pixels.",
584  np.count_nonzero(vignMask))
lsst.cp.pipe.cpCombine.CalibCombineConnections
Definition: cpCombine.py:96
lsst::afw::image
lsst.cp.pipe.cpCombine.CalibStatsTask
Definition: cpCombine.py:62
lsst.cp.pipe.cpCombine.CalibCombineTask.combine
def combine(self, target, expList, stats)
Definition: cpCombine.py:410
lsst.cp.pipe.cpCombine.CalibCombineTask.run
def run(self, inputExps, inputScales=None, inputDims=None)
Definition: cpCombine.py:241
lsst.cp.pipe.cpCombine.CalibCombineTask.getSize
def getSize(self, dimList)
Definition: cpCombine.py:365
lsst.cp.pipe.cpCombine.CalibCombineTask.interpolateNans
def interpolateNans(self, exp)
Definition: cpCombine.py:505
lsst.cp.pipe.cpCombine.CalibCombineConnections.inputScales
inputScales
Definition: cpCombine.py:104
lsst.cp.pipe.cpCombine.VignetteExposure
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)
Definition: cpCombine.py:526
lsst.cp.pipe.cpCombine.CalibCombineTask.getDimensions
def getDimensions(self, expList)
Definition: cpCombine.py:349
lsst.cp.pipe.cpCombine.CalibCombineConnections.outputData
outputData
Definition: cpCombine.py:112
lsst.cp.pipe.cpCombine.CalibCombineTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: cpCombine.py:232
lsst.cp.pipe.cpCombine.CalibCombineTask.applyScale
def applyScale(self, exposure, scale=None)
Definition: cpCombine.py:388
lsst.cp.pipe.cpCombine.CalibCombineTask.__init__
def __init__(self, **kwargs)
Definition: cpCombine.py:228
lsst::pex::config
lsst.cp.pipe.cpCombine.CalibCombineTask
Definition: cpCombine.py:223
lsst::geom
lsst.cp.pipe.cpCombine.CalibStatsConfig
Definition: cpCombine.py:37
lsst.cp.pipe.cpCombine.CalibCombineTask.combineHeaders
def combineHeaders(self, expList, calib, calibType="CALIB", scales=None)
Definition: cpCombine.py:425
lsst.cp.pipe.cpCombine.CalibStatsTask.run
def run(self, exposureOrImage)
Definition: cpCombine.py:69
lsst::afw::math
Point< double, 2 >
lsst::pipe::base
lsst::log
lsst.cp.pipe.cpCombine.CalibCombineConnections.__init__
def __init__(self, *config=None)
Definition: cpCombine.py:119
lsst::pipe::base::connectionTypes
lsst.cp.pipe.cpCombine.CalibCombineConfig
Definition: cpCombine.py:149