lsst.cp.pipe  20.0.0-6-g20dd8cc
cpCombine.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import numpy as np
22 import time
23 
24 import lsst.pex.config as pexConfig
25 import lsst.pipe.base as pipeBase
27 import lsst.afw.math as afwMath
28 import lsst.afw.image as afwImage
29 
30 from lsst.geom import Point2D
31 from lsst.log import Log
32 from astro_metadata_translator import merge_headers, ObservationGroup
33 from astro_metadata_translator.serialize import dates_to_fits
34 
35 
36 # CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
37 class CalibStatsConfig(pexConfig.Config):
38  """Parameters controlling the measurement of background statistics.
39  """
40  stat = pexConfig.Field(
41  dtype=int,
42  default=int(afwMath.MEANCLIP),
43  doc="Statistic to use to estimate background (from lsst.afw.math)",
44  )
45  clip = pexConfig.Field(
46  dtype=float,
47  default=3.0,
48  doc="Clipping threshold for background",
49  )
50  nIter = pexConfig.Field(
51  dtype=int,
52  default=3,
53  doc="Clipping iterations for background",
54  )
55  mask = pexConfig.ListField(
56  dtype=str,
57  default=["DETECTED", "BAD", "NO_DATA"],
58  doc="Mask planes to reject",
59  )
60 
61 
62 class CalibStatsTask(pipeBase.Task):
63  """Measure statistics on the background
64 
65  This can be useful for scaling the background, e.g., for flats and fringe frames.
66  """
67  ConfigClass = CalibStatsConfig
68 
69  def run(self, exposureOrImage):
70  """Measure a particular statistic on an image (of some sort).
71 
72  Parameters
73  ----------
74  exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75  Exposure or image to calculate statistics on.
76 
77  Returns
78  -------
79  results : float
80  Resulting statistic value.
81  """
82  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83  afwImage.Mask.getPlaneBitMask(self.config.mask))
84  try:
85  image = exposureOrImage.getMaskedImage()
86  except Exception:
87  try:
88  image = exposureOrImage.getImage()
89  except Exception:
90  image = exposureOrImage
91 
92  return afwMath.makeStatistics(image, self.config.stat, stats).getValue()
93 
94 
95 class CalibCombineConnections(pipeBase.PipelineTaskConnections,
96  dimensions=("instrument", "detector")):
97  inputExps = cT.Input(
98  name="cpInputs",
99  doc="Input pre-processed exposures to combine.",
100  storageClass="ExposureF",
101  dimensions=("instrument", "detector", "exposure"),
102  multiple=True,
103  )
104  inputScales = cT.Input(
105  name="cpScales",
106  doc="Input scale factors to use.",
107  storageClass="StructuredDataDict",
108  dimensions=("instrument", ),
109  multiple=False,
110  )
111 
112  outputData = cT.Output(
113  name="cpProposal",
114  doc="Output combined proposed calibration.",
115  storageClass="ExposureF",
116  dimensions=("instrument", "detector"),
117  )
118 
119  def __init__(self, *, config=None):
120  super().__init__(config=config)
121 
122  if config and len(config.calibrationDimensions) != 0:
123  newDimensions = tuple(config.calibrationDimensions)
124  newOutputData = cT.Output(
125  name=self.outputData.name,
126  doc=self.outputData.doc,
127  storageClass=self.outputData.storageClass,
128  dimensions=self.allConnections['outputData'].dimensions + newDimensions
129  )
130  self.dimensions.update(config.calibrationDimensions)
131  self.outputData = newOutputData
132 
133  if config.exposureScaling == 'InputList':
134  newInputScales = cT.PrerequisiteInput(
135  name=self.inputScales.name,
136  doc=self.inputScales.doc,
137  storageClass=self.inputScales.storageClass,
138  dimensions=self.allConnections['inputScales'].dimensions + newDimensions
139  )
140  self.dimensions.update(config.calibrationDimensions)
141  self.inputScales = newInputScales
142 
143 
144 # CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
145 class CalibCombineConfig(pipeBase.PipelineTaskConfig,
146  pipelineConnections=CalibCombineConnections):
147  """Configuration for combining calib exposures.
148  """
149  calibrationType = pexConfig.Field(
150  dtype=str,
151  default="calibration",
152  doc="Name of calibration to be generated.",
153  )
154  calibrationDimensions = pexConfig.ListField(
155  dtype=str,
156  default=[],
157  doc="List of updated dimensions to append to output."
158  )
159 
160  exposureScaling = pexConfig.ChoiceField(
161  dtype=str,
162  allowed={
163  "None": "No scaling used.",
164  "ExposureTime": "Scale inputs by their exposure time.",
165  "DarkTime": "Scale inputs by their dark time.",
166  "MeanStats": "Scale inputs based on their mean values.",
167  "InputList": "Scale inputs based on a list of values.",
168  },
169  default=None,
170  doc="Scaling to be applied to each input exposure.",
171  )
172  scalingLevel = pexConfig.ChoiceField(
173  dtype=str,
174  allowed={
175  "DETECTOR": "Scale by detector.",
176  "AMP": "Scale by amplifier.",
177  },
178  default="DETECTOR",
179  doc="Region to scale.",
180  )
181  maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
182  dtype=int,
183  default=5,
184  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
185  )
186 
187  doVignette = pexConfig.Field(
188  dtype=bool,
189  default=False,
190  doc="Copy vignette polygon to output and censor vignetted pixels?"
191  )
192 
193  mask = pexConfig.ListField(
194  dtype=str,
195  default=["SAT", "DETECTED", "INTRP"],
196  doc="Mask planes to respect",
197  )
198  combine = pexConfig.Field(
199  dtype=int,
200  default=int(afwMath.MEANCLIP),
201  doc="Statistic to use for combination (from lsst.afw.math)",
202  )
203  clip = pexConfig.Field(
204  dtype=float,
205  default=3.0,
206  doc="Clipping threshold for combination",
207  )
208  nIter = pexConfig.Field(
209  dtype=int,
210  default=3,
211  doc="Clipping iterations for combination",
212  )
213  stats = pexConfig.ConfigurableField(
214  target=CalibStatsTask,
215  doc="Background statistics configuration",
216  )
217 
218 
219 class CalibCombineTask(pipeBase.PipelineTask,
220  pipeBase.CmdLineTask):
221  """Task to combine calib exposures."""
222  ConfigClass = CalibCombineConfig
223  _DefaultName = 'cpCombine'
224 
225  def __init__(self, **kwargs):
226  super().__init__(**kwargs)
227  self.makeSubtask("stats")
228 
229  def runQuantum(self, butlerQC, inputRefs, outputRefs):
230  inputs = butlerQC.get(inputRefs)
231 
232  dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
233  inputs['inputDims'] = dimensions
234 
235  outputs = self.run(**inputs)
236  butlerQC.put(outputs, outputRefs)
237 
238  def run(self, inputExps, inputScales=None, inputDims=None):
239  """Combine calib exposures for a single detector.
240 
241  Parameters
242  ----------
243  inputExps : `list` [`lsst.afw.image.Exposure`]
244  Input list of exposures to combine.
245  inputScales : `dict` [`dict` [`dict` [`float`]]], optional
246  Dictionary of scales, indexed by detector (`int`),
247  amplifier (`int`), and exposure (`int`). Used for
248  'inputList' scaling.
249  inputDims : `list` [`dict`]
250  List of dictionaries of input data dimensions/values.
251  Each list entry should contain:
252 
253  ``"exposure"``
254  exposure id value (`int`)
255  ``"detector"``
256  detector id value (`int`)
257 
258  Returns
259  -------
260  combinedExp : `lsst.afw.image.Exposure`
261  Final combined exposure generated from the inputs.
262 
263  Raises
264  ------
265  RuntimeError
266  Raised if no input data is found. Also raised if
267  config.exposureScaling == InputList, and a necessary scale
268  was not found.
269  """
270  width, height = self.getDimensions(inputExps)
271  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
272  afwImage.Mask.getPlaneBitMask(self.config.mask))
273  numExps = len(inputExps)
274  if numExps < 1:
275  raise RuntimeError("No valid input data")
276  if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
277  stats.setCalcErrorFromInputVariance(True)
278 
279  # Create output exposure for combined data.
280  combined = afwImage.MaskedImageF(width, height)
281  combinedExp = afwImage.makeExposure(combined)
282 
283  # Apply scaling:
284  expScales = []
285  if inputDims is None:
286  inputDims = [dict() for i in inputExps]
287 
288  for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
289  scale = 1.0
290  if exp is None:
291  self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims)
292  continue
293 
294  if self.config.exposureScaling == "ExposureTime":
295  scale = exp.getInfo().getVisitInfo().getExposureTime()
296  elif self.config.exposureScaling == "DarkTime":
297  scale = exp.getInfo().getVisitInfo().getDarkTime()
298  elif self.config.exposureScaling == "MeanStats":
299  scale = self.stats.run(exp)
300  elif self.config.exposureScaling == "InputList":
301  visitId = dims.get('exposure', None)
302  detectorId = dims.get('detector', None)
303  if visitId is None or detectorId is None:
304  raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
305  if detectorId not in inputScales['expScale']:
306  raise RuntimeError(f"Could not identify a scaling for input {index}"
307  f" detector {detectorId}")
308 
309  if self.config.scalingLevel == "DETECTOR":
310  if visitId not in inputScales['expScale'][detectorId]:
311  raise RuntimeError(f"Could not identify a scaling for input {index}"
312  f"detector {detectorId} visit {visitId}")
313  scale = inputScales['expScale'][detectorId][visitId]
314  elif self.config.scalingLevel == 'AMP':
315  scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
316  for amp in exp.getDetector()]
317  else:
318  raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
319  elif self.config.exposureScaling == 'None':
320  scale = 1.0
321  else:
322  raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
323 
324  expScales.append(scale)
325  self.log.info("Scaling input %d by %s", index, scale)
326  self.applyScale(exp, scale)
327 
328  self.combine(combined, inputExps, stats)
329 
330  self.interpolateNans(combined)
331 
332  if self.config.doVignette:
333  polygon = inputExps[0].getInfo().getValidPolygon()
334  VignetteExposure(combined, polygon=polygon, doUpdateMask=True,
335  doSetValue=True, vignetteValue=0.0)
336 
337  # Combine headers
338  self.combineHeaders(inputExps, combinedExp,
339  calibType=self.config.calibrationType, scales=expScales)
340 
341  # Return
342  return pipeBase.Struct(
343  outputData=combinedExp,
344  )
345 
346  def getDimensions(self, expList):
347  """Get dimensions of the inputs.
348 
349  Parameters
350  ----------
351  expList : `list` [`lsst.afw.image.Exposure`]
352  Exps to check the sizes of.
353 
354  Returns
355  -------
356  width, height : `int`
357  Unique set of input dimensions.
358  """
359  dimList = [exp.getDimensions() for exp in expList if exp is not None]
360  return self.getSize(dimList)
361 
362  def getSize(self, dimList):
363  """Determine a consistent size, given a list of image sizes.
364 
365  Parameters
366  -----------
367  dimList : iterable of `tuple` (`int`, `int`)
368  List of dimensions.
369 
370  Raises
371  ------
372  RuntimeError
373  If input dimensions are inconsistent.
374 
375  Returns
376  --------
377  width, height : `int`
378  Common dimensions.
379  """
380  dim = set((w, h) for w, h in dimList)
381  if len(dim) != 1:
382  raise RuntimeError("Inconsistent dimensions: %s" % dim)
383  return dim.pop()
384 
385  def applyScale(self, exposure, scale=None):
386  """Apply scale to input exposure.
387 
388  This implementation applies a flux scaling: the input exposure is
389  divided by the provided scale.
390 
391  Parameters
392  ----------
393  exposure : `lsst.afw.image.Exposure`
394  Exposure to scale.
395  scale : `float` or `list` [`float`], optional
396  Constant scale to divide the exposure by.
397  """
398  if scale is not None:
399  mi = exposure.getMaskedImage()
400  if isinstance(scale, list):
401  for amp, ampScale in zip(exposure.getDetector(), scale):
402  ampIm = mi[amp.getBBox()]
403  ampIm /= ampScale
404  else:
405  mi /= scale
406 
407  def combine(self, target, expList, stats):
408  """Combine multiple images.
409 
410  Parameters
411  ----------
412  target : `lsst.afw.image.Exposure`
413  Output exposure to construct.
414  expList : `list` [`lsst.afw.image.Exposure`]
415  Input exposures to combine.
416  stats : `lsst.afw.math.StatisticsControl`
417  Control explaining how to combine the input images.
418  """
419  images = [img.getMaskedImage() for img in expList if img is not None]
420  afwMath.statisticsStack(target, images, afwMath.Property(self.config.combine), stats)
421 
422  def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
423  """Combine input headers to determine the set of common headers,
424  supplemented by calibration inputs.
425 
426  Parameters
427  ----------
428  expList : `list` of `lsst.afw.image.Exposure`
429  Input list of exposures to combine.
430  calib : `lsst.afw.image.Exposure`
431  Output calibration to construct headers for.
432  calibType: `str`, optional
433  OBSTYPE the output should claim.
434  scales: `list` of `float`, optional
435  Scale values applied to each input to record.
436 
437  Returns
438  -------
439  header : `lsst.daf.base.PropertyList`
440  Constructed header.
441  """
442  # Header
443  header = calib.getMetadata()
444  header.set("OBSTYPE", calibType)
445 
446  # Keywords we care about
447  comments = {"TIMESYS": "Time scale for all dates",
448  "DATE-OBS": "Start date of earliest input observation",
449  "MJD-OBS": "[d] Start MJD of earliest input observation",
450  "DATE-END": "End date of oldest input observation",
451  "MJD-END": "[d] End MJD of oldest input observation",
452  "MJD-AVG": "[d] MJD midpoint of all input observations",
453  "DATE-AVG": "Midpoint date of all input observations"}
454 
455  # Creation date
456  now = time.localtime()
457  calibDate = time.strftime("%Y-%m-%d", now)
458  calibTime = time.strftime("%X %Z", now)
459  header.set("CALIB_CREATE_DATE", calibDate)
460  header.set("CALIB_CREATE_TIME", calibTime)
461 
462  # Merge input headers
463  inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
464  merged = merge_headers(inputHeaders, mode='drop')
465  for k, v in merged.items():
466  if k not in header:
467  md = expList[0].getMetadata()
468  comment = md.getComment(k) if k in md else None
469  header.set(k, v, comment=comment)
470 
471  # Construct list of visits
472  visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
473  for i, visit in enumerate(visitInfoList):
474  if visit is None:
475  continue
476  header.set("CPP_INPUT_%d" % (i,), visit.getExposureId())
477  header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
478  header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
479  if scales is not None:
480  header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
481 
482  # Not yet working: DM-22302
483  # Create an observation group so we can add some standard headers
484  # independent of the form in the input files.
485  # Use try block in case we are dealing with unexpected data headers
486  try:
487  group = ObservationGroup(visitInfoList, pedantic=False)
488  except Exception:
489  self.log.warn("Exception making an obs group for headers. Continuing.")
490  # Fall back to setting a DATE-OBS from the calibDate
491  dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
492  comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
493  else:
494  oldest, newest = group.extremes()
495  dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
496 
497  for k, v in dateCards.items():
498  header.set(k, v, comment=comments.get(k, None))
499 
500  return header
501 
502  def interpolateNans(self, exp):
503  """Interpolate over NANs in the combined image.
504 
505  NANs can result from masked areas on the CCD. We don't want them getting
506  into our science images, so we replace them with the median of the image.
507 
508  Parameters
509  ----------
510  exp : `lsst.afw.image.Exposure`
511  Exp to check for NaNs.
512  """
513  array = exp.getImage().getArray()
514  bad = np.isnan(array)
515 
516  median = np.median(array[np.logical_not(bad)])
517  count = np.sum(np.logical_not(bad))
518  array[bad] = median
519  if count > 0:
520  self.log.warn("Found %s NAN pixels", count)
521 
522 
523 def VignetteExposure(exposure, polygon=None,
524  doUpdateMask=True, maskPlane='BAD',
525  doSetValue=False, vignetteValue=0.0,
526  log=None):
527  """Apply vignetted polygon to image pixels.
528 
529  Parameters
530  ----------
531  exposure : `lsst.afw.image.Exposure`
532  Image to be updated.
533  doUpdateMask : `bool`, optional
534  Update the exposure mask for vignetted area?
535  maskPlane : `str`, optional,
536  Mask plane to assign.
537  doSetValue : `bool`, optional
538  Set image value for vignetted area?
539  vignetteValue : `float`, optional
540  Value to assign.
541  log : `lsst.log.Log`, optional
542  Log to write to.
543 
544  Raises
545  ------
546  RuntimeError
547  Raised if no valid polygon exists.
548  """
549  polygon = polygon if polygon else exposure.getInfo().getValidPolygon()
550  if not polygon:
551  raise RuntimeError("Could not find valid polygon!")
552  log = log if log else Log.getLogger(__name__.partition(".")[2])
553 
554  fullyIlluminated = True
555  for corner in exposure.getBBox().getCorners():
556  if not polygon.contains(Point2D(corner)):
557  fullyIlluminated = False
558 
559  log.info("Exposure is fully illuminated? %s", fullyIlluminated)
560 
561  if not fullyIlluminated:
562  # Scan pixels.
563  mask = exposure.getMask()
564  numPixels = mask.getBBox().getArea()
565 
566  xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
567  np.arange(0, mask.getHeight(), dtype=int))
568 
569  vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in
570  zip(xx.reshape(numPixels), yy.reshape(numPixels))])
571  vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
572 
573  if doUpdateMask:
574  bitMask = mask.getPlaneBitMask(maskPlane)
575  maskArray = mask.getArray()
576  maskArray[vignMask] |= bitMask
577  if doSetValue:
578  imageArray = exposure.getImage().getArray()
579  imageArray[vignMask] = vignetteValue
580  log.info("Exposure contains %d vignetted pixels.",
581  np.count_nonzero(vignMask))
lsst.cp.pipe.cpCombine.CalibCombineConnections
Definition: cpCombine.py:96
lsst::afw::image
lsst.cp.pipe.cpCombine.CalibStatsTask
Definition: cpCombine.py:62
lsst.cp.pipe.cpCombine.CalibCombineTask.combine
def combine(self, target, expList, stats)
Definition: cpCombine.py:407
lsst.cp.pipe.cpCombine.CalibCombineTask.run
def run(self, inputExps, inputScales=None, inputDims=None)
Definition: cpCombine.py:238
lsst.cp.pipe.cpCombine.CalibCombineTask.getSize
def getSize(self, dimList)
Definition: cpCombine.py:362
lsst.cp.pipe.cpCombine.CalibCombineTask.interpolateNans
def interpolateNans(self, exp)
Definition: cpCombine.py:502
lsst.cp.pipe.cpCombine.CalibCombineConnections.inputScales
inputScales
Definition: cpCombine.py:104
lsst.cp.pipe.cpCombine.VignetteExposure
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)
Definition: cpCombine.py:523
lsst.cp.pipe.cpCombine.CalibCombineTask.getDimensions
def getDimensions(self, expList)
Definition: cpCombine.py:346
lsst.cp.pipe.cpCombine.CalibCombineConnections.outputData
outputData
Definition: cpCombine.py:112
lsst.cp.pipe.cpCombine.CalibCombineTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: cpCombine.py:229
lsst.cp.pipe.cpCombine.CalibCombineTask.applyScale
def applyScale(self, exposure, scale=None)
Definition: cpCombine.py:385
lsst.cp.pipe.cpCombine.CalibCombineTask.__init__
def __init__(self, **kwargs)
Definition: cpCombine.py:225
lsst.cp.pipe.cpCombine.CalibCombineTask
Definition: cpCombine.py:220
lsst::geom
lsst.cp.pipe.cpCombine.CalibStatsConfig
Definition: cpCombine.py:37
lsst.cp.pipe.cpCombine.CalibCombineTask.combineHeaders
def combineHeaders(self, expList, calib, calibType="CALIB", scales=None)
Definition: cpCombine.py:422
lsst.cp.pipe.cpCombine.CalibStatsTask.run
def run(self, exposureOrImage)
Definition: cpCombine.py:69
lsst::afw::math
Point< double, 2 >
lsst::pipe::base
lsst::log
lsst.cp.pipe.cpCombine.CalibCombineConnections.__init__
def __init__(self, *config=None)
Definition: cpCombine.py:119
lsst::pipe::base::connectionTypes
lsst.cp.pipe.cpCombine.CalibCombineConfig
Definition: cpCombine.py:146