lsst.pipe.tasks  17.0.1-11-gf64fdeb0+3
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  optional=True,
46  doc="Maximum number of iterations of forward modeling.",
47  default=None,
48  )
49  minNumIter = pexConfig.Field(
50  dtype=int,
51  optional=True,
52  doc="Minimum number of iterations of forward modeling.",
53  default=None,
54  )
55  convergenceThreshold = pexConfig.Field(
56  dtype=float,
57  doc="Target relative change in convergence between iterations of forward modeling.",
58  default=0.001,
59  )
60  useConvergence = pexConfig.Field(
61  dtype=bool,
62  doc="Use convergence test as a forward modeling end condition?"
63  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
64  default=True,
65  )
66  baseGain = pexConfig.Field(
67  dtype=float,
68  optional=True,
69  doc="Relative weight to give the new solution vs. the last solution when updating the model."
70  "A value of 1.0 gives equal weight to both solutions."
71  "Small values imply slower convergence of the solution, but can "
72  "help prevent overshooting and failures in the fit."
73  "If ``baseGain`` is None, a conservative gain "
74  "will be calculated from the number of subfilters. ",
75  default=None,
76  )
77  useProgressiveGain = pexConfig.Field(
78  dtype=bool,
79  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
80  "When calculating the next gain, we use up to 5 previous gains and convergence values."
81  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
82  default=True,
83  )
84  doAirmassWeight = pexConfig.Field(
85  dtype=bool,
86  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
87  default=False,
88  )
89  modelWeightsWidth = pexConfig.Field(
90  dtype=float,
91  doc="Width of the region around detected sources to include in the DcrModel.",
92  default=3,
93  )
94  useModelWeights = pexConfig.Field(
95  dtype=bool,
96  doc="Width of the region around detected sources to include in the DcrModel.",
97  default=True,
98  )
99  splitSubfilters = pexConfig.Field(
100  dtype=bool,
101  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
102  "Instead of at the midpoint",
103  default=True,
104  )
105  regularizeModelIterations = pexConfig.Field(
106  dtype=float,
107  doc="Maximum relative change of the model allowed between iterations."
108  "Set to zero to disable.",
109  default=2.,
110  )
111  regularizeModelFrequency = pexConfig.Field(
112  dtype=float,
113  doc="Maximum relative change of the model allowed between subfilters."
114  "Set to zero to disable.",
115  default=4.,
116  )
117  convergenceMaskPlanes = pexConfig.ListField(
118  dtype=str,
119  default=["DETECTED"],
120  doc="Mask planes to use to calculate convergence."
121  )
122  regularizationWidth = pexConfig.Field(
123  dtype=int,
124  default=2,
125  doc="Minimum radius of a region to include in regularization, in pixels."
126  )
127  imageInterpOrder = pexConfig.Field(
128  dtype=int,
129  doc="The order of the spline interpolation used to shift the image plane.",
130  default=3,
131  )
132 
133  def setDefaults(self):
134  CompareWarpAssembleCoaddConfig.setDefaults(self)
135  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
136  self.doNImage = True
137  self.warpType = 'direct'
138  self.assembleStaticSkyModel.warpType = self.warpType
139  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
140  self.assembleStaticSkyModel.doNImage = False
141  self.assembleStaticSkyModel.doWrite = False
142 
143 
145  """Assemble DCR coadded images from a set of warps.
146 
147  Attributes
148  ----------
149  bufferSize : `int`
150  The number of pixels to grow each subregion by to allow for DCR.
151 
152  Notes
153  -----
154  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
155  Warps (also called coadded temporary exposures), including the effects of
156  Differential Chromatic Refraction (DCR).
157  For full details of the mathematics and algorithm, please see
158  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
159 
160  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
161  each subfilter used in the iterative calculation.
162  It begins by dividing the bandpass-defining filter into N equal bandwidth
163  "subfilters", and divides the flux in each pixel from an initial coadd
164  equally into each as a "dcrModel". Because the airmass and parallactic
165  angle of each individual exposure is known, we can calculate the shift
166  relative to the center of the band in each subfilter due to DCR. For each
167  exposure we apply this shift as a linear transformation to the dcrModels
168  and stack the results to produce a DCR-matched exposure. The matched
169  exposures are subtracted from the input exposures to produce a set of
170  residual images, and these residuals are reverse shifted for each
171  exposures' subfilters and stacked. The shifted and stacked residuals are
172  added to the dcrModels to produce a new estimate of the flux in each pixel
173  within each subfilter. The dcrModels are solved for iteratively, which
174  continues until the solution from a new iteration improves by less than
175  a set percentage, or a maximum number of iterations is reached.
176  Two forms of regularization are employed to reduce unphysical results.
177  First, the new solution is averaged with the solution from the previous
178  iteration, which mitigates oscillating solutions where the model
179  overshoots with alternating very high and low values.
180  Second, a common degeneracy when the data have a limited range of airmass or
181  parallactic angle values is for one subfilter to be fit with very low or
182  negative values, while another subfilter is fit with very high values. This
183  typically appears in the form of holes next to sources in one subfilter,
184  and corresponding extended wings in another. Because each subfilter has
185  a narrow bandwidth we assume that physical sources that are above the noise
186  level will not vary in flux by more than a factor of `frequencyClampFactor`
187  between subfilters, and pixels that have flux deviations larger than that
188  factor will have the excess flux distributed evenly among all subfilters.
189  """
190 
191  ConfigClass = DcrAssembleCoaddConfig
192  _DefaultName = "dcrAssembleCoadd"
193 
194  @pipeBase.timeMethod
195  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
196  """Assemble a coadd from a set of warps.
197 
198  Coadd a set of Warps. Compute weights to be applied to each Warp and
199  find scalings to match the photometric zeropoint to a reference Warp.
200  Assemble the Warps using run method.
201  Forward model chromatic effects across multiple subfilters,
202  and subtract from the input Warps to build sets of residuals.
203  Use the residuals to construct a new ``DcrModel`` for each subfilter,
204  and iterate until the model converges.
205  Interpolate over NaNs and optionally write the coadd to disk.
206  Return the coadded exposure.
207 
208  Parameters
209  ----------
210  dataRef : `lsst.daf.persistence.ButlerDataRef`
211  Data reference defining the patch for coaddition and the
212  reference Warp
213  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
214  List of data references to warps. Data to be coadded will be
215  selected from this list based on overlap with the patch defined by
216  the data reference.
217 
218  Returns
219  -------
220  results : `lsst.pipe.base.Struct`
221  The Struct contains the following fields:
222 
223  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
224  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
225  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
226  - ``dcrNImages``: `list` of exposure count images for each subfilter
227  """
228  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
229  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
230 
231  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
232  warpRefList=warpRefList)
233  if results is None:
234  skyInfo = self.getSkyInfo(dataRef)
235  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
236  skyInfo.patchInfo.getIndex())
237  return
238  for subfilter in range(self.config.dcrNumSubfilters):
239  self.processResults(results.dcrCoadds[subfilter], dataRef)
240  if self.config.doWrite:
241  self.log.info("Persisting dcrCoadd")
242  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
243  numSubfilters=self.config.dcrNumSubfilters)
244  if self.config.doNImage and results.dcrNImages is not None:
245  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
246  numSubfilters=self.config.dcrNumSubfilters)
247 
248  return results
249 
250  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
251  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
252 
253  Sets the property ``bufferSize``.
254 
255  Parameters
256  ----------
257  templateCoadd : `lsst.afw.image.ExposureF`
258  The initial coadd exposure before accounting for DCR.
259  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
260  The data references to the input warped exposures.
261  weightList : `list` of `float`
262  The weight to give each input exposure in the coadd
263  Will be modified in place if ``doAirmassWeight`` is set.
264 
265  Returns
266  -------
267  dcrModels : `lsst.pipe.tasks.DcrModel`
268  Best fit model of the true sky after correcting chromatic effects.
269 
270  Raises
271  ------
272  NotImplementedError
273  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
274  """
275  filterInfo = templateCoadd.getFilter()
276  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
277  raise NotImplementedError("No minimum/maximum wavelength information found"
278  " in the filter definition! Please add lambdaMin and lambdaMax"
279  " to the Mapper class in your obs package.")
280  tempExpName = self.getTempExpDatasetName(self.warpType)
281  dcrShifts = []
282  airmassList = {}
283  angleList = {}
284  for visitNum, tempExpRef in enumerate(tempExpRefList):
285  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
286  airmass = visitInfo.getBoresightAirmass()
287  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
288  airmassList[tempExpRef.dataId["visit"]] = airmass
289  angleList[tempExpRef.dataId["visit"]] = parallacticAngle
290  if self.config.doAirmassWeight:
291  weightList[visitNum] *= airmass
292  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
293  filterInfo, self.config.dcrNumSubfilters))))
294  self.log.info("Selected airmasses:\n%s", airmassList)
295  self.log.info("Selected parallactic angles:\n%s", angleList)
296  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
297  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
298  self.config.dcrNumSubfilters,
299  filterInfo=filterInfo,
300  psf=templateCoadd.getPsf())
301  return dcrModels
302 
303  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
304  supplementaryData=None):
305  """Assemble the coadd.
306 
307  Requires additional inputs Struct ``supplementaryData`` to contain a
308  ``templateCoadd`` that serves as the model of the static sky.
309 
310  Find artifacts and apply them to the warps' masks creating a list of
311  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
312  Then pass these alternative masks to the base class's assemble method.
313 
314  Divide the ``templateCoadd`` evenly between each subfilter of a
315  ``DcrModel`` as the starting best estimate of the true wavelength-
316  dependent sky. Forward model the ``DcrModel`` using the known
317  chromatic effects in each subfilter and calculate a convergence metric
318  based on how well the modeled template matches the input warps. If
319  the convergence has not yet reached the desired threshold, then shift
320  and stack the residual images to build a new ``DcrModel``. Apply
321  conditioning to prevent oscillating solutions between iterations or
322  between subfilters.
323 
324  Once the ``DcrModel`` reaches convergence or the maximum number of
325  iterations has been reached, fill the metadata for each subfilter
326  image and make them proper ``coaddExposure``s.
327 
328  Parameters
329  ----------
330  skyInfo : `lsst.pipe.base.Struct`
331  Patch geometry information, from getSkyInfo
332  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
333  The data references to the input warped exposures.
334  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
335  The image scalars correct for the zero point of the exposures.
336  weightList : `list` of `float`
337  The weight to give each input exposure in the coadd
338  supplementaryData : `lsst.pipe.base.Struct`
339  Result struct returned by ``makeSupplementaryData`` with components:
340 
341  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
342 
343  Returns
344  -------
345  result : `lsst.pipe.base.Struct`
346  Result struct with components:
347 
348  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
349  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
350  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
351  - ``dcrNImages``: `list` of exposure count images for each subfilter
352  """
353  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
354  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
355  templateCoadd = supplementaryData.templateCoadd
356  baseMask = templateCoadd.mask.clone()
357  # The variance plane is for each subfilter
358  # and should be proportionately lower than the full-band image
359  baseVariance = templateCoadd.variance.clone()
360  baseVariance /= self.config.dcrNumSubfilters
361  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
362  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
363  templateCoadd.setMask(baseMask)
364  badMaskPlanes = self.config.badMaskPlanes[:]
365  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
366  # This is because pixels in observations that are significantly affect by DCR
367  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
368  # but those are necessary to constrain the DCR model.
369  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
370 
371  stats = self.prepareStats(mask=badPixelMask)
372  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
373  if self.config.doNImage:
374  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, tempExpRefList,
375  spanSetMaskList, stats.ctrl)
376  nImage = afwImage.ImageU(skyInfo.bbox)
377  # Note that this nImage will be a factor of dcrNumSubfilters higher than
378  # the nImage returned by assembleCoadd for most pixels. This is because each
379  # subfilter may have a different nImage, and fractional values are not allowed.
380  for dcrNImage in dcrNImages:
381  nImage += dcrNImage
382  else:
383  dcrNImages = None
384 
385  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
386  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
387  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
388  subIter = 0
389  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
390  modelIter = 0
391  subIter += 1
392  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
393  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
394  dcrBBox = afwGeom.Box2I(subBBox)
395  dcrBBox.grow(self.bufferSize)
396  dcrBBox.clip(dcrModels.bbox)
397  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
398  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, tempExpRefList,
399  imageScalerList, spanSetMaskList)
400  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
401  tempExpRefList, weightList, stats.ctrl)
402  self.log.info("Initial convergence : %s", convergenceMetric)
403  convergenceList = [convergenceMetric]
404  gainList = []
405  convergenceCheck = 1.
406  refImage = templateCoadd.image
407  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
408  gain = self.calculateGain(convergenceList, gainList)
409  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, tempExpRefList,
410  stats.ctrl, convergenceMetric, baseMask, gain,
411  modelWeights, refImage, dcrWeights)
412  if self.config.useConvergence:
413  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
414  tempExpRefList, weightList, stats.ctrl)
415  if convergenceMetric == 0:
416  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
417  "most likely due to there being no valid data in the region.",
418  skyInfo.patchInfo.getIndex(), subIter)
419  break
420  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
421  if (convergenceCheck < 0) & (modelIter > minNumIter):
422  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
423  "iterations or desired convergence improvement of %s."
424  " Divergence: %s",
425  skyInfo.patchInfo.getIndex(), subIter,
426  self.config.convergenceThreshold, convergenceCheck)
427  break
428  convergenceList.append(convergenceMetric)
429  if modelIter > maxNumIter:
430  if self.config.useConvergence:
431  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
432  "before reaching desired convergence improvement of %s."
433  " Final convergence improvement: %s",
434  skyInfo.patchInfo.getIndex(), subIter,
435  self.config.convergenceThreshold, convergenceCheck)
436  break
437 
438  if self.config.useConvergence:
439  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
440  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
441  modelIter += 1
442  else:
443  if self.config.useConvergence:
444  self.log.info("Coadd patch %s subregion %s finished with "
445  "convergence metric %s after %s iterations",
446  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
447  else:
448  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
449  skyInfo.patchInfo.getIndex(), subIter, modelIter)
450  if self.config.useConvergence and convergenceMetric > 0:
451  self.log.info("Final convergence improvement was %.4f%% overall",
452  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
453 
454  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
455  calibration=self.scaleZeroPoint.getPhotoCalib(),
456  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
457  mask=baseMask,
458  variance=baseVariance)
459  coaddExposure = self.stackCoadd(dcrCoadds)
460  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
461  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
462 
463  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
464  """Calculate the number of exposures contributing to each subfilter.
465 
466  Parameters
467  ----------
468  dcrModels : `lsst.pipe.tasks.DcrModel`
469  Best fit model of the true sky after correcting chromatic effects.
470  bbox : `lsst.afw.geom.box.Box2I`
471  Bounding box of the patch to coadd.
472  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
473  The data references to the input warped exposures.
474  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
475  Each element of the `dict` contains the new mask plane name
476  (e.g. "CLIPPED and/or "NO_DATA") as the key,
477  and the list of SpanSets to apply to the mask.
478  statsCtrl : `lsst.afw.math.StatisticsControl`
479  Statistics control object for coadd
480 
481  Returns
482  -------
483  dcrNImages : `list` of `lsst.afw.image.ImageU`
484  List of exposure count images for each subfilter
485  dcrWeights : `list` of `lsst.afw.image.ImageF`
486  Per-pixel weights for each subfilter.
487  Equal to the number of unmasked images contributing to each pixel.
488  """
489  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
490  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
491  tempExpName = self.getTempExpDatasetName(self.warpType)
492  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
493  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
494  visitInfo = exposure.getInfo().getVisitInfo()
495  wcs = exposure.getInfo().getWcs()
496  mask = exposure.mask
497  if altMaskSpans is not None:
498  self.applyAltMaskPlanes(mask, altMaskSpans)
499  weightImage = np.zeros_like(exposure.image.array)
500  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
501  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
502  for dcr, dcrNImage, dcrWeight in zip(dcrShift, dcrNImages, dcrWeights):
503  # Note that we use the same interpolation for the weights and the images.
504  shiftedWeights = applyDcr(weightImage, dcr, useInverse=True,
505  order=self.config.imageInterpOrder)
506  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
507  dcrWeight.array += shiftedWeights
508  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
509  weightsThreshold = 1.
510  goodPix = dcrWeights[0].array > weightsThreshold
511  for weights in dcrWeights[1:]:
512  goodPix = (weights.array > weightsThreshold) & goodPix
513  for subfilter in range(self.config.dcrNumSubfilters):
514  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
515  dcrWeights[subfilter].array[~goodPix] = 0.
516  dcrNImages[subfilter].array[~goodPix] = 0
517  return (dcrNImages, dcrWeights)
518 
519  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, tempExpRefList,
520  statsCtrl, convergenceMetric,
521  baseMask, gain, modelWeights, refImage, dcrWeights):
522  """Assemble the DCR coadd for a sub-region.
523 
524  Build a DCR-matched template for each input exposure, then shift the
525  residuals according to the DCR in each subfilter.
526  Stack the shifted residuals and apply them as a correction to the
527  solution from the previous iteration.
528  Restrict the new model solutions from varying by more than a factor of
529  `modelClampFactor` from the last solution, and additionally restrict the
530  individual subfilter models from varying by more than a factor of
531  `frequencyClampFactor` from their average.
532  Finally, mitigate potentially oscillating solutions by averaging the new
533  solution with the solution from the previous iteration, weighted by
534  their convergence metric.
535 
536  Parameters
537  ----------
538  dcrModels : `lsst.pipe.tasks.DcrModel`
539  Best fit model of the true sky after correcting chromatic effects.
540  subExposures : `dict` of `lsst.afw.image.ExposureF`
541  The pre-loaded exposures for the current subregion.
542  bbox : `lsst.afw.geom.box.Box2I`
543  Bounding box of the subregion to coadd.
544  dcrBBox : `lsst.afw.geom.box.Box2I`
545  Sub-region of the coadd which includes a buffer to allow for DCR.
546  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
547  The data references to the input warped exposures.
548  statsCtrl : `lsst.afw.math.StatisticsControl`
549  Statistics control object for coadd
550  convergenceMetric : `float`
551  Quality of fit metric for the matched templates of the input images.
552  baseMask : `lsst.afw.image.Mask`
553  Mask of the initial template coadd.
554  gain : `float`, optional
555  Relative weight to give the new solution when updating the model.
556  modelWeights : `numpy.ndarray` or `float`
557  A 2D array of weight values that tapers smoothly to zero away from detected sources.
558  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
559  refImage : `lsst.afw.image.Image`
560  A reference image used to supply the default pixel values.
561  dcrWeights : `list` of `lsst.afw.image.Image`
562  Per-pixel weights for each subfilter.
563  Equal to the number of unmasked images contributing to each pixel.
564  """
565  residualGeneratorList = []
566 
567  for tempExpRef in tempExpRefList:
568  exposure = subExposures[tempExpRef.dataId["visit"]]
569  visitInfo = exposure.getInfo().getVisitInfo()
570  wcs = exposure.getInfo().getWcs()
571  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
572  order=self.config.imageInterpOrder,
573  splitSubfilters=self.config.splitSubfilters)
574  residual = exposure.image.array - templateImage.array
575  # Note that the variance plane here is used to store weights, not the actual variance
576  residual *= exposure.variance.array
577  # The residuals are stored as a list of generators.
578  # This allows the residual for a given subfilter and exposure to be created
579  # on the fly, instead of needing to store them all in memory.
580  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
581 
582  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
583  mask=baseMask, gain=gain,
584  modelWeights=modelWeights,
585  refImage=refImage,
586  dcrWeights=dcrWeights)
587  dcrModels.assign(dcrSubModelOut, bbox)
588 
589  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
590  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
591 
592  Parameters
593  ----------
594  residual : `numpy.ndarray`
595  The residual masked image for one exposure,
596  after subtracting the matched template
597  visitInfo : `lsst.afw.image.VisitInfo`
598  Metadata for the exposure.
599  wcs : `lsst.afw.geom.SkyWcs`
600  Coordinate system definition (wcs) for the exposure.
601  filterInfo : `lsst.afw.image.Filter`
602  The filter definition, set in the current instruments' obs package.
603  Required for any calculation of DCR, including making matched templates.
604 
605  Yields
606  ------
607  residualImage : `numpy.ndarray`
608  The residual image for the next subfilter, shifted for DCR.
609  """
610  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
611  for dcr in dcrShift:
612  yield applyDcr(residual, dcr, useInverse=True, order=self.config.imageInterpOrder)
613 
614  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
615  mask, gain, modelWeights, refImage, dcrWeights):
616  """Calculate a new DcrModel from a set of image residuals.
617 
618  Parameters
619  ----------
620  dcrModels : `lsst.pipe.tasks.DcrModel`
621  Current model of the true sky after correcting chromatic effects.
622  residualGeneratorList : `generator` of `numpy.ndarray`
623  The residual image for the next subfilter, shifted for DCR.
624  dcrBBox : `lsst.afw.geom.box.Box2I`
625  Sub-region of the coadd which includes a buffer to allow for DCR.
626  statsCtrl : `lsst.afw.math.StatisticsControl`
627  Statistics control object for coadd
628  mask : `lsst.afw.image.Mask`
629  Mask to use for each new model image.
630  gain : `float`
631  Relative weight to give the new solution when updating the model.
632  modelWeights : `numpy.ndarray` or `float`
633  A 2D array of weight values that tapers smoothly to zero away from detected sources.
634  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
635  refImage : `lsst.afw.image.Image`
636  A reference image used to supply the default pixel values.
637  dcrWeights : `list` of `lsst.afw.image.Image`
638  Per-pixel weights for each subfilter.
639  Equal to the number of unmasked images contributing to each pixel.
640 
641  Returns
642  -------
643  dcrModel : `lsst.pipe.tasks.DcrModel`
644  New model of the true sky after correcting chromatic effects.
645  """
646  newModelImages = []
647  for subfilter, model in enumerate(dcrModels):
648  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
649  residual = np.sum(residualsList, axis=0)
650  residual *= dcrWeights[subfilter][dcrBBox].array
651  # `MaskedImage`s only support in-place addition, so rename for readability
652  newModel = model[dcrBBox].clone()
653  newModel.array += residual
654  # Catch any invalid values
655  badPixels = ~np.isfinite(newModel.array)
656  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
657  if self.config.regularizeModelIterations > 0:
658  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
659  self.config.regularizeModelIterations,
660  self.config.regularizationWidth)
661  newModelImages.append(newModel)
662  if self.config.regularizeModelFrequency > 0:
663  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
664  self.config.regularizeModelFrequency,
665  self.config.regularizationWidth,
666  mask=mask)
667  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
668  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
669  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
670  dcrModels.mask, dcrModels.variance)
671 
672  def calculateConvergence(self, dcrModels, subExposures, bbox, tempExpRefList, weightList, statsCtrl):
673  """Calculate a quality of fit metric for the matched templates.
674 
675  Parameters
676  ----------
677  dcrModels : `lsst.pipe.tasks.DcrModel`
678  Best fit model of the true sky after correcting chromatic effects.
679  subExposures : `dict` of `lsst.afw.image.ExposureF`
680  The pre-loaded exposures for the current subregion.
681  bbox : `lsst.afw.geom.box.Box2I`
682  Sub-region to coadd
683  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
684  The data references to the input warped exposures.
685  weightList : `list` of `float`
686  The weight to give each input exposure in the coadd
687  statsCtrl : `lsst.afw.math.StatisticsControl`
688  Statistics control object for coadd
689 
690  Returns
691  -------
692  convergenceMetric : `float`
693  Quality of fit metric for all input exposures, within the sub-region
694  """
695  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
696  nSigma = 3.
697  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
698  bufferSize=self.bufferSize)
699  if np.max(significanceImage) == 0:
700  significanceImage += 1.
701  weight = 0
702  metric = 0.
703  metricList = {}
704  for tempExpRef, expWeight in zip(tempExpRefList, weightList):
705  exposure = subExposures[tempExpRef.dataId["visit"]][bbox]
706  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
707  metric += singleMetric
708  metricList[tempExpRef.dataId["visit"]] = singleMetric
709  weight += 1.
710  self.log.info("Individual metrics:\n%s", metricList)
711  return 1.0 if weight == 0.0 else metric/weight
712 
713  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
714  """Calculate a quality of fit metric for a single matched template.
715 
716  Parameters
717  ----------
718  dcrModels : `lsst.pipe.tasks.DcrModel`
719  Best fit model of the true sky after correcting chromatic effects.
720  exposure : `lsst.afw.image.ExposureF`
721  The input warped exposure to evaluate.
722  significanceImage : `numpy.ndarray`
723  Array of weights for each pixel corresponding to its significance
724  for the convergence calculation.
725  statsCtrl : `lsst.afw.math.StatisticsControl`
726  Statistics control object for coadd
727 
728  Returns
729  -------
730  convergenceMetric : `float`
731  Quality of fit metric for one exposure, within the sub-region.
732  """
733  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
734  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, order=self.config.imageInterpOrder)
735  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
736  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
737 
738  finitePixels = np.isfinite(diffVals)
739  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
740  convergeMaskPixels = exposure.mask.array & convergeMask > 0
741  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
742  if np.sum(usePixels) == 0:
743  metric = 0.
744  else:
745  diffUse = diffVals[usePixels]
746  refUse = refVals[usePixels]
747  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
748  return metric
749 
750  def stackCoadd(self, dcrCoadds):
751  """Add a list of sub-band coadds together.
752 
753  Parameters
754  ----------
755  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
756  A list of coadd exposures, each exposure containing
757  the model for one subfilter.
758 
759  Returns
760  -------
761  coaddExposure : `lsst.afw.image.ExposureF`
762  A single coadd exposure that is the sum of the sub-bands.
763  """
764  coaddExposure = dcrCoadds[0].clone()
765  for coadd in dcrCoadds[1:]:
766  coaddExposure.maskedImage += coadd.maskedImage
767  return coaddExposure
768 
769  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
770  mask=None, variance=None):
771  """Create a list of coadd exposures from a list of masked images.
772 
773  Parameters
774  ----------
775  dcrModels : `lsst.pipe.tasks.DcrModel`
776  Best fit model of the true sky after correcting chromatic effects.
777  skyInfo : `lsst.pipe.base.Struct`
778  Patch geometry information, from getSkyInfo
779  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
780  The data references to the input warped exposures.
781  weightList : `list` of `float`
782  The weight to give each input exposure in the coadd
783  calibration : `lsst.afw.Image.PhotoCalib`, optional
784  Scale factor to set the photometric calibration of an exposure.
785  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
786  A record of the observations that are included in the coadd.
787  mask : `lsst.afw.image.Mask`, optional
788  Optional mask to override the values in the final coadd.
789  variance : `lsst.afw.image.Image`, optional
790  Optional variance plane to override the values in the final coadd.
791 
792  Returns
793  -------
794  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
795  A list of coadd exposures, each exposure containing
796  the model for one subfilter.
797  """
798  dcrCoadds = []
799  for model in dcrModels:
800  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
801  if calibration is not None:
802  coaddExposure.setPhotoCalib(calibration)
803  if coaddInputs is not None:
804  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
805  # Set the metadata for the coadd, including PSF and aperture corrections.
806  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
807  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
808  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
809  maskedImage.image = model
810  maskedImage.mask = dcrModels.mask
811  maskedImage.variance = dcrModels.variance
812  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
813  if mask is not None:
814  coaddExposure.setMask(mask)
815  if variance is not None:
816  coaddExposure.setVariance(variance)
817  dcrCoadds.append(coaddExposure)
818  return dcrCoadds
819 
820  def calculateGain(self, convergenceList, gainList):
821  """Calculate the gain to use for the current iteration.
822 
823  After calculating a new DcrModel, each value is averaged with the
824  value in the corresponding pixel from the previous iteration. This
825  reduces oscillating solutions that iterative techniques are plagued by,
826  and speeds convergence. By far the biggest changes to the model
827  happen in the first couple iterations, so we can also use a more
828  aggressive gain later when the model is changing slowly.
829 
830  Parameters
831  ----------
832  convergenceList : `list` of `float`
833  The quality of fit metric from each previous iteration.
834  gainList : `list` of `float`
835  The gains used in each previous iteration: appended with the new
836  gain value.
837  Gains are numbers between ``self.config.baseGain`` and 1.
838 
839  Returns
840  -------
841  gain : `float`
842  Relative weight to give the new solution when updating the model.
843  A value of 1.0 gives equal weight to both solutions.
844 
845  Raises
846  ------
847  ValueError
848  If ``len(convergenceList) != len(gainList)+1``.
849  """
850  nIter = len(convergenceList)
851  if nIter != len(gainList) + 1:
852  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
853  % (len(convergenceList), len(gainList)))
854 
855  if self.config.baseGain is None:
856  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
857  # The more subfilters being modeled, the lower the gain should be.
858  baseGain = 1./(self.config.dcrNumSubfilters - 1)
859  else:
860  baseGain = self.config.baseGain
861 
862  if self.config.useProgressiveGain and nIter > 2:
863  # To calculate the best gain to use, compare the past gains that have been used
864  # with the resulting convergences to estimate the best gain to use.
865  # Algorithmically, this is a Kalman filter.
866  # If forward modeling proceeds perfectly, the convergence metric should
867  # asymptotically approach a final value.
868  # We can estimate that value from the measured changes in convergence
869  # weighted by the gains used in each previous iteration.
870  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
871  for i in range(nIter - 1)]
872  # The convergence metric is strictly positive, so if the estimated final convergence is
873  # less than zero, force it to zero.
874  estFinalConv = np.array(estFinalConv)
875  estFinalConv[estFinalConv < 0] = 0
876  # Because the estimate may slowly change over time, only use the most recent measurements.
877  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
878  lastGain = gainList[-1]
879  lastConv = convergenceList[-2]
880  newConv = convergenceList[-1]
881  # The predicted convergence is the value we would get if the new model calculated
882  # in the previous iteration was perfect. Recall that the updated model that is
883  # actually used is the gain-weighted average of the new and old model,
884  # so the convergence would be similarly weighted.
885  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
886  # If the measured and predicted convergence are very close, that indicates
887  # that our forward model is accurate and we can use a more aggressive gain
888  # If the measured convergence is significantly worse (or better!) than predicted,
889  # that indicates that the model is not converging as expected and
890  # we should use a more conservative gain.
891  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
892  newGain = 1 - abs(delta)
893  # Average the gains to prevent oscillating solutions.
894  newGain = (newGain + lastGain)/2.
895  gain = max(baseGain, newGain)
896  else:
897  gain = baseGain
898  gainList.append(gain)
899  return gain
900 
901  def calculateModelWeights(self, dcrModels, dcrBBox):
902  """Build an array that smoothly tapers to 0 away from detected sources.
903 
904  Parameters
905  ----------
906  dcrModels : `lsst.pipe.tasks.DcrModel`
907  Best fit model of the true sky after correcting chromatic effects.
908  dcrBBox : `lsst.afw.geom.box.Box2I`
909  Sub-region of the coadd which includes a buffer to allow for DCR.
910 
911  Returns
912  -------
913  weights : `numpy.ndarray` or `float`
914  A 2D array of weight values that tapers smoothly to zero away from detected sources.
915  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
916 
917  Raises
918  ------
919  ValueError
920  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
921  """
922  if not self.config.useModelWeights:
923  return 1.0
924  if self.config.modelWeightsWidth < 0:
925  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
926  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
927  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
928  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
929  weights[convergeMaskPixels] = 1.
930  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
931  weights /= np.max(weights)
932  return weights
933 
934  def applyModelWeights(self, modelImages, refImage, modelWeights):
935  """Smoothly replace model pixel values with those from a
936  reference at locations away from detected sources.
937 
938  Parameters
939  ----------
940  modelImages : `list` of `lsst.afw.image.Image`
941  The new DCR model images from the current iteration.
942  The values will be modified in place.
943  refImage : `lsst.afw.image.MaskedImage`
944  A reference image used to supply the default pixel values.
945  modelWeights : `numpy.ndarray` or `float`
946  A 2D array of weight values that tapers smoothly to zero away from detected sources.
947  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
948  """
949  if self.config.useModelWeights:
950  for model in modelImages:
951  model.array *= modelWeights
952  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
953 
954  def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList):
955  """Pre-load sub-regions of a list of exposures.
956 
957  Parameters
958  ----------
959  bbox : `lsst.afw.geom.box.Box2I`
960  Sub-region to coadd
961  statsCtrl : `lsst.afw.math.StatisticsControl`
962  Statistics control object for coadd
963  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
964  The data references to the input warped exposures.
965  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
966  The image scalars correct for the zero point of the exposures.
967  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
968  Each element is dict with keys = mask plane name to add the spans to
969 
970  Returns
971  -------
972  subExposures : `dict`
973  The `dict` keys are the visit IDs,
974  and the values are `lsst.afw.image.ExposureF`
975  The pre-loaded exposures for the current subregion.
976  The variance plane contains weights, and not the variance
977  """
978  tempExpName = self.getTempExpDatasetName(self.warpType)
979  zipIterables = zip(tempExpRefList, imageScalerList, spanSetMaskList)
980  subExposures = {}
981  for tempExpRef, imageScaler, altMaskSpans in zipIterables:
982  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
983  if altMaskSpans is not None:
984  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
985  imageScaler.scaleMaskedImage(exposure.maskedImage)
986  # Note that the variance plane here is used to store weights, not the actual variance
987  exposure.variance.array[:, :] = 0.
988  # Set the weight of unmasked pixels to 1.
989  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
990  # Set the image value of masked pixels to zero.
991  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
992  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
993  subExposures[tempExpRef.dataId["visit"]] = exposure
994  return subExposures
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, tempExpRefList, weightList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, tempExpRefList, statsCtrl, convergenceMetric, baseMask, gain, modelWeights, refImage, dcrWeights)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, mask, gain, modelWeights, refImage, dcrWeights)
def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList)