lsst.cp.pipe  21.0.0-22-gd45243f+75349ef5b2
cpCombine.py
Go to the documentation of this file.
1 # This file is part of cp_pipe.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 import numpy as np
22 import time
23 
24 import lsst.pex.config as pexConfig
25 import lsst.pipe.base as pipeBase
27 import lsst.afw.math as afwMath
28 import lsst.afw.image as afwImage
29 
30 from lsst.geom import Point2D
31 from lsst.log import Log
32 from astro_metadata_translator import merge_headers, ObservationGroup
33 from astro_metadata_translator.serialize import dates_to_fits
34 
35 
36 # CalibStatsConfig/CalibStatsTask from pipe_base/constructCalibs.py
37 class CalibStatsConfig(pexConfig.Config):
38  """Parameters controlling the measurement of background statistics.
39  """
40  stat = pexConfig.Field(
41  dtype=str,
42  default='MEANCLIP',
43  doc="Statistic name to use to estimate background (from lsst.afw.math)",
44  )
45  clip = pexConfig.Field(
46  dtype=float,
47  default=3.0,
48  doc="Clipping threshold for background",
49  )
50  nIter = pexConfig.Field(
51  dtype=int,
52  default=3,
53  doc="Clipping iterations for background",
54  )
55  mask = pexConfig.ListField(
56  dtype=str,
57  default=["DETECTED", "BAD", "NO_DATA"],
58  doc="Mask planes to reject",
59  )
60 
61 
62 class CalibStatsTask(pipeBase.Task):
63  """Measure statistics on the background
64 
65  This can be useful for scaling the background, e.g., for flats and fringe frames.
66  """
67  ConfigClass = CalibStatsConfig
68 
69  def run(self, exposureOrImage):
70  """Measure a particular statistic on an image (of some sort).
71 
72  Parameters
73  ----------
74  exposureOrImage : `lsst.afw.image.Exposure`, `lsst.afw.image.MaskedImage`, or `lsst.afw.image.Image`
75  Exposure or image to calculate statistics on.
76 
77  Returns
78  -------
79  results : float
80  Resulting statistic value.
81  """
82  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
83  afwImage.Mask.getPlaneBitMask(self.config.mask))
84  try:
85  image = exposureOrImage.getMaskedImage()
86  except Exception:
87  try:
88  image = exposureOrImage.getImage()
89  except Exception:
90  image = exposureOrImage
91  statType = afwMath.stringToStatisticsProperty(self.config.stat)
92  return afwMath.makeStatistics(image, statType, stats).getValue()
93 
94 
95 class CalibCombineConnections(pipeBase.PipelineTaskConnections,
96  dimensions=("instrument", "detector")):
97  inputExps = cT.Input(
98  name="cpInputs",
99  doc="Input pre-processed exposures to combine.",
100  storageClass="Exposure",
101  dimensions=("instrument", "detector", "exposure"),
102  multiple=True,
103  )
104  inputScales = cT.Input(
105  name="cpScales",
106  doc="Input scale factors to use.",
107  storageClass="StructuredDataDict",
108  dimensions=("instrument", ),
109  multiple=False,
110  )
111 
112  outputData = cT.Output(
113  name="cpProposal",
114  doc="Output combined proposed calibration to be validated and certified..",
115  storageClass="ExposureF",
116  dimensions=("instrument", "detector"),
117  isCalibration=True,
118  )
119 
120  def __init__(self, *, config=None):
121  super().__init__(config=config)
122 
123  if config and config.exposureScaling != 'InputList':
124  self.inputs.discard("inputScales")
125 
126 
127 # CalibCombineConfig/CalibCombineTask from pipe_base/constructCalibs.py
128 class CalibCombineConfig(pipeBase.PipelineTaskConfig,
129  pipelineConnections=CalibCombineConnections):
130  """Configuration for combining calib exposures.
131  """
132  calibrationType = pexConfig.Field(
133  dtype=str,
134  default="calibration",
135  doc="Name of calibration to be generated.",
136  )
137 
138  exposureScaling = pexConfig.ChoiceField(
139  dtype=str,
140  allowed={
141  "Unity": "Do not scale inputs. Scale factor is 1.0.",
142  "ExposureTime": "Scale inputs by their exposure time.",
143  "DarkTime": "Scale inputs by their dark time.",
144  "MeanStats": "Scale inputs based on their mean values.",
145  "InputList": "Scale inputs based on a list of values.",
146  },
147  default="Unity",
148  doc="Scaling to be applied to each input exposure.",
149  )
150  scalingLevel = pexConfig.ChoiceField(
151  dtype=str,
152  allowed={
153  "DETECTOR": "Scale by detector.",
154  "AMP": "Scale by amplifier.",
155  },
156  default="DETECTOR",
157  doc="Region to scale.",
158  )
159  maxVisitsToCalcErrorFromInputVariance = pexConfig.Field(
160  dtype=int,
161  default=5,
162  doc="Maximum number of visits to estimate variance from input variance, not per-pixel spread",
163  )
164 
165  doVignette = pexConfig.Field(
166  dtype=bool,
167  default=False,
168  doc="Copy vignette polygon to output and censor vignetted pixels?"
169  )
170 
171  mask = pexConfig.ListField(
172  dtype=str,
173  default=["SAT", "DETECTED", "INTRP"],
174  doc="Mask planes to respect",
175  )
176  combine = pexConfig.Field(
177  dtype=str,
178  default='MEANCLIP',
179  doc="Statistic name to use for combination (from lsst.afw.math)",
180  )
181  clip = pexConfig.Field(
182  dtype=float,
183  default=3.0,
184  doc="Clipping threshold for combination",
185  )
186  nIter = pexConfig.Field(
187  dtype=int,
188  default=3,
189  doc="Clipping iterations for combination",
190  )
191  stats = pexConfig.ConfigurableField(
192  target=CalibStatsTask,
193  doc="Background statistics configuration",
194  )
195 
196 
197 class CalibCombineTask(pipeBase.PipelineTask,
198  pipeBase.CmdLineTask):
199  """Task to combine calib exposures."""
200  ConfigClass = CalibCombineConfig
201  _DefaultName = 'cpCombine'
202 
203  def __init__(self, **kwargs):
204  super().__init__(**kwargs)
205  self.makeSubtask("stats")
206 
207  def runQuantum(self, butlerQC, inputRefs, outputRefs):
208  inputs = butlerQC.get(inputRefs)
209 
210  dimensions = [exp.dataId.byName() for exp in inputRefs.inputExps]
211  inputs['inputDims'] = dimensions
212 
213  outputs = self.runrun(**inputs)
214  butlerQC.put(outputs, outputRefs)
215 
216  def run(self, inputExps, inputScales=None, inputDims=None):
217  """Combine calib exposures for a single detector.
218 
219  Parameters
220  ----------
221  inputExps : `list` [`lsst.afw.image.Exposure`]
222  Input list of exposures to combine.
223  inputScales : `dict` [`dict` [`dict` [`float`]]], optional
224  Dictionary of scales, indexed by detector (`int`),
225  amplifier (`int`), and exposure (`int`). Used for
226  'inputList' scaling.
227  inputDims : `list` [`dict`]
228  List of dictionaries of input data dimensions/values.
229  Each list entry should contain:
230 
231  ``"exposure"``
232  exposure id value (`int`)
233  ``"detector"``
234  detector id value (`int`)
235 
236  Returns
237  -------
238  combinedExp : `lsst.afw.image.Exposure`
239  Final combined exposure generated from the inputs.
240 
241  Raises
242  ------
243  RuntimeError
244  Raised if no input data is found. Also raised if
245  config.exposureScaling == InputList, and a necessary scale
246  was not found.
247  """
248  width, height = self.getDimensionsgetDimensions(inputExps)
249  stats = afwMath.StatisticsControl(self.config.clip, self.config.nIter,
250  afwImage.Mask.getPlaneBitMask(self.config.mask))
251  numExps = len(inputExps)
252  if numExps < 1:
253  raise RuntimeError("No valid input data")
254  if numExps < self.config.maxVisitsToCalcErrorFromInputVariance:
255  stats.setCalcErrorFromInputVariance(True)
256 
257  # Create output exposure for combined data.
258  combined = afwImage.MaskedImageF(width, height)
259  combinedExp = afwImage.makeExposure(combined)
260 
261  # Apply scaling:
262  expScales = []
263  if inputDims is None:
264  inputDims = [dict() for i in inputExps]
265 
266  for index, (exp, dims) in enumerate(zip(inputExps, inputDims)):
267  scale = 1.0
268  if exp is None:
269  self.log.warn("Input %d is None (%s); unable to scale exp.", index, dims)
270  continue
271 
272  if self.config.exposureScaling == "ExposureTime":
273  scale = exp.getInfo().getVisitInfo().getExposureTime()
274  elif self.config.exposureScaling == "DarkTime":
275  scale = exp.getInfo().getVisitInfo().getDarkTime()
276  elif self.config.exposureScaling == "MeanStats":
277  scale = self.stats.run(exp)
278  elif self.config.exposureScaling == "InputList":
279  visitId = dims.get('exposure', None)
280  detectorId = dims.get('detector', None)
281  if visitId is None or detectorId is None:
282  raise RuntimeError(f"Could not identify scaling for input {index} ({dims})")
283  if detectorId not in inputScales['expScale']:
284  raise RuntimeError(f"Could not identify a scaling for input {index}"
285  f" detector {detectorId}")
286 
287  if self.config.scalingLevel == "DETECTOR":
288  if visitId not in inputScales['expScale'][detectorId]:
289  raise RuntimeError(f"Could not identify a scaling for input {index}"
290  f"detector {detectorId} visit {visitId}")
291  scale = inputScales['expScale'][detectorId][visitId]
292  elif self.config.scalingLevel == 'AMP':
293  scale = [inputScales['expScale'][detectorId][amp.getName()][visitId]
294  for amp in exp.getDetector()]
295  else:
296  raise RuntimeError(f"Unknown scaling level: {self.config.scalingLevel}")
297  elif self.config.exposureScaling == 'Unity':
298  scale = 1.0
299  else:
300  raise RuntimeError(f"Unknown scaling type: {self.config.exposureScaling}.")
301 
302  expScales.append(scale)
303  self.log.info("Scaling input %d by %s", index, scale)
304  self.applyScaleapplyScale(exp, scale)
305 
306  self.combinecombine(combined, inputExps, stats)
307 
308  self.interpolateNansinterpolateNans(combined)
309 
310  if self.config.doVignette:
311  polygon = inputExps[0].getInfo().getValidPolygon()
312  VignetteExposure(combined, polygon=polygon, doUpdateMask=True,
313  doSetValue=True, vignetteValue=0.0)
314 
315  # Combine headers
316  self.combineHeaderscombineHeaders(inputExps, combinedExp,
317  calibType=self.config.calibrationType, scales=expScales)
318 
319  # Return
320  return pipeBase.Struct(
321  outputData=combinedExp,
322  )
323 
324  def getDimensions(self, expList):
325  """Get dimensions of the inputs.
326 
327  Parameters
328  ----------
329  expList : `list` [`lsst.afw.image.Exposure`]
330  Exps to check the sizes of.
331 
332  Returns
333  -------
334  width, height : `int`
335  Unique set of input dimensions.
336  """
337  dimList = [exp.getDimensions() for exp in expList if exp is not None]
338  return self.getSizegetSize(dimList)
339 
340  def getSize(self, dimList):
341  """Determine a consistent size, given a list of image sizes.
342 
343  Parameters
344  -----------
345  dimList : iterable of `tuple` (`int`, `int`)
346  List of dimensions.
347 
348  Raises
349  ------
350  RuntimeError
351  If input dimensions are inconsistent.
352 
353  Returns
354  --------
355  width, height : `int`
356  Common dimensions.
357  """
358  dim = set((w, h) for w, h in dimList)
359  if len(dim) != 1:
360  raise RuntimeError("Inconsistent dimensions: %s" % dim)
361  return dim.pop()
362 
363  def applyScale(self, exposure, scale=None):
364  """Apply scale to input exposure.
365 
366  This implementation applies a flux scaling: the input exposure is
367  divided by the provided scale.
368 
369  Parameters
370  ----------
371  exposure : `lsst.afw.image.Exposure`
372  Exposure to scale.
373  scale : `float` or `list` [`float`], optional
374  Constant scale to divide the exposure by.
375  """
376  if scale is not None:
377  mi = exposure.getMaskedImage()
378  if isinstance(scale, list):
379  for amp, ampScale in zip(exposure.getDetector(), scale):
380  ampIm = mi[amp.getBBox()]
381  ampIm /= ampScale
382  else:
383  mi /= scale
384 
385  def combine(self, target, expList, stats):
386  """Combine multiple images.
387 
388  Parameters
389  ----------
390  target : `lsst.afw.image.Exposure`
391  Output exposure to construct.
392  expList : `list` [`lsst.afw.image.Exposure`]
393  Input exposures to combine.
394  stats : `lsst.afw.math.StatisticsControl`
395  Control explaining how to combine the input images.
396  """
397  images = [img.getMaskedImage() for img in expList if img is not None]
398  combineType = afwMath.stringToStatisticsProperty(self.config.combine)
399  afwMath.statisticsStack(target, images, combineType, stats)
400 
401  def combineHeaders(self, expList, calib, calibType="CALIB", scales=None):
402  """Combine input headers to determine the set of common headers,
403  supplemented by calibration inputs.
404 
405  Parameters
406  ----------
407  expList : `list` of `lsst.afw.image.Exposure`
408  Input list of exposures to combine.
409  calib : `lsst.afw.image.Exposure`
410  Output calibration to construct headers for.
411  calibType: `str`, optional
412  OBSTYPE the output should claim.
413  scales: `list` of `float`, optional
414  Scale values applied to each input to record.
415 
416  Returns
417  -------
418  header : `lsst.daf.base.PropertyList`
419  Constructed header.
420  """
421  # Header
422  header = calib.getMetadata()
423  header.set("OBSTYPE", calibType)
424 
425  # Keywords we care about
426  comments = {"TIMESYS": "Time scale for all dates",
427  "DATE-OBS": "Start date of earliest input observation",
428  "MJD-OBS": "[d] Start MJD of earliest input observation",
429  "DATE-END": "End date of oldest input observation",
430  "MJD-END": "[d] End MJD of oldest input observation",
431  "MJD-AVG": "[d] MJD midpoint of all input observations",
432  "DATE-AVG": "Midpoint date of all input observations"}
433 
434  # Creation date
435  now = time.localtime()
436  calibDate = time.strftime("%Y-%m-%d", now)
437  calibTime = time.strftime("%X %Z", now)
438  header.set("CALIB_CREATE_DATE", calibDate)
439  header.set("CALIB_CREATE_TIME", calibTime)
440 
441  # Merge input headers
442  inputHeaders = [exp.getMetadata() for exp in expList if exp is not None]
443  merged = merge_headers(inputHeaders, mode='drop')
444  for k, v in merged.items():
445  if k not in header:
446  md = expList[0].getMetadata()
447  comment = md.getComment(k) if k in md else None
448  header.set(k, v, comment=comment)
449 
450  # Construct list of visits
451  visitInfoList = [exp.getInfo().getVisitInfo() for exp in expList if exp is not None]
452  for i, visit in enumerate(visitInfoList):
453  if visit is None:
454  continue
455  header.set("CPP_INPUT_%d" % (i,), visit.getExposureId())
456  header.set("CPP_INPUT_DATE_%d" % (i,), str(visit.getDate()))
457  header.set("CPP_INPUT_EXPT_%d" % (i,), visit.getExposureTime())
458  if scales is not None:
459  header.set("CPP_INPUT_SCALE_%d" % (i,), scales[i])
460 
461  # Not yet working: DM-22302
462  # Create an observation group so we can add some standard headers
463  # independent of the form in the input files.
464  # Use try block in case we are dealing with unexpected data headers
465  try:
466  group = ObservationGroup(visitInfoList, pedantic=False)
467  except Exception:
468  self.log.warn("Exception making an obs group for headers. Continuing.")
469  # Fall back to setting a DATE-OBS from the calibDate
470  dateCards = {"DATE-OBS": "{}T00:00:00.00".format(calibDate)}
471  comments["DATE-OBS"] = "Date of start of day of calibration midpoint"
472  else:
473  oldest, newest = group.extremes()
474  dateCards = dates_to_fits(oldest.datetime_begin, newest.datetime_end)
475 
476  for k, v in dateCards.items():
477  header.set(k, v, comment=comments.get(k, None))
478 
479  return header
480 
481  def interpolateNans(self, exp):
482  """Interpolate over NANs in the combined image.
483 
484  NANs can result from masked areas on the CCD. We don't want them getting
485  into our science images, so we replace them with the median of the image.
486 
487  Parameters
488  ----------
489  exp : `lsst.afw.image.Exposure`
490  Exp to check for NaNs.
491  """
492  array = exp.getImage().getArray()
493  bad = np.isnan(array)
494 
495  median = np.median(array[np.logical_not(bad)])
496  count = np.sum(np.logical_not(bad))
497  array[bad] = median
498  if count > 0:
499  self.log.warn("Found %s NAN pixels", count)
500 
501 
502 # Create versions of the Connections, Config, and Task that support filter constraints.
504  dimensions=("instrument", "detector", "physical_filter")):
505  inputScales = cT.Input(
506  name="cpFilterScales",
507  doc="Input scale factors to use.",
508  storageClass="StructuredDataDict",
509  dimensions=("instrument", "physical_filter"),
510  multiple=False,
511  )
512 
513  outputData = cT.Output(
514  name="cpFilterProposal",
515  doc="Output combined proposed calibration to be validated and certified.",
516  storageClass="ExposureF",
517  dimensions=("instrument", "detector", "physical_filter"),
518  isCalibration=True,
519  )
520 
521  def __init__(self, *, config=None):
522  super().__init__(config=config)
523 
524  if config and config.exposureScaling != 'InputList':
525  self.inputs.discard("inputScales")
526 
527 
529  pipelineConnections=CalibCombineByFilterConnections):
530  pass
531 
532 
534  """Task to combine calib exposures."""
535  ConfigClass = CalibCombineByFilterConfig
536  _DefaultName = 'cpFilterCombine'
537  pass
538 
539 
540 def VignetteExposure(exposure, polygon=None,
541  doUpdateMask=True, maskPlane='BAD',
542  doSetValue=False, vignetteValue=0.0,
543  log=None):
544  """Apply vignetted polygon to image pixels.
545 
546  Parameters
547  ----------
548  exposure : `lsst.afw.image.Exposure`
549  Image to be updated.
550  doUpdateMask : `bool`, optional
551  Update the exposure mask for vignetted area?
552  maskPlane : `str`, optional,
553  Mask plane to assign.
554  doSetValue : `bool`, optional
555  Set image value for vignetted area?
556  vignetteValue : `float`, optional
557  Value to assign.
558  log : `lsst.log.Log`, optional
559  Log to write to.
560 
561  Raises
562  ------
563  RuntimeError
564  Raised if no valid polygon exists.
565  """
566  polygon = polygon if polygon else exposure.getInfo().getValidPolygon()
567  if not polygon:
568  raise RuntimeError("Could not find valid polygon!")
569  log = log if log else Log.getLogger(__name__.partition(".")[2])
570 
571  fullyIlluminated = True
572  for corner in exposure.getBBox().getCorners():
573  if not polygon.contains(Point2D(corner)):
574  fullyIlluminated = False
575 
576  log.info("Exposure is fully illuminated? %s", fullyIlluminated)
577 
578  if not fullyIlluminated:
579  # Scan pixels.
580  mask = exposure.getMask()
581  numPixels = mask.getBBox().getArea()
582 
583  xx, yy = np.meshgrid(np.arange(0, mask.getWidth(), dtype=int),
584  np.arange(0, mask.getHeight(), dtype=int))
585 
586  vignMask = np.array([not polygon.contains(Point2D(x, y)) for x, y in
587  zip(xx.reshape(numPixels), yy.reshape(numPixels))])
588  vignMask = vignMask.reshape(mask.getHeight(), mask.getWidth())
589 
590  if doUpdateMask:
591  bitMask = mask.getPlaneBitMask(maskPlane)
592  maskArray = mask.getArray()
593  maskArray[vignMask] |= bitMask
594  if doSetValue:
595  imageArray = exposure.getImage().getArray()
596  imageArray[vignMask] = vignetteValue
597  log.info("Exposure contains %d vignetted pixels.",
598  np.count_nonzero(vignMask))
def applyScale(self, exposure, scale=None)
Definition: cpCombine.py:363
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: cpCombine.py:207
def run(self, inputExps, inputScales=None, inputDims=None)
Definition: cpCombine.py:216
def combine(self, target, expList, stats)
Definition: cpCombine.py:385
def combineHeaders(self, expList, calib, calibType="CALIB", scales=None)
Definition: cpCombine.py:401
def run(self, exposureOrImage)
Definition: cpCombine.py:69
def VignetteExposure(exposure, polygon=None, doUpdateMask=True, maskPlane='BAD', doSetValue=False, vignetteValue=0.0, log=None)
Definition: cpCombine.py:543