lsst.pipe.tasks  16.0-35-g79496433+1
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from scipy import ndimage
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  doc="Maximum number of iterations of forward modeling.",
46  default=8,
47  )
48  minNumIter = pexConfig.Field(
49  dtype=int,
50  doc="Minimum number of iterations of forward modeling.",
51  default=3,
52  )
53  convergenceThreshold = pexConfig.Field(
54  dtype=float,
55  doc="Target relative change in convergence between iterations of forward modeling.",
56  default=0.001,
57  )
58  useConvergence = pexConfig.Field(
59  dtype=bool,
60  doc="Use convergence test as a forward modeling end condition?"
61  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
62  default=True,
63  )
64  baseGain = pexConfig.Field(
65  dtype=float,
66  doc="Relative weight to give the new solution vs. the last solution when updating the model."
67  "A value of 1.0 gives equal weight to both solutions."
68  "Small values imply slower convergence of the solution, but can "
69  "help prevent overshooting and failures in the fit."
70  "If ``baseGain`` is None, a conservative gain "
71  "will be calculated from the number of subfilters. ",
72  default=None,
73  )
74  useProgressiveGain = pexConfig.Field(
75  dtype=bool,
76  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
77  "When calculating the next gain, we use up to 5 previous gains and convergence values."
78  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
79  default=True,
80  )
81  doAirmassWeight = pexConfig.Field(
82  dtype=bool,
83  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
84  default=True,
85  )
86  modelWeightsWidth = pexConfig.Field(
87  dtype=float,
88  doc="Width of the region around detected sources to include in the DcrModel.",
89  default=3,
90  )
91  useModelWeights = pexConfig.Field(
92  dtype=bool,
93  doc="Width of the region around detected sources to include in the DcrModel.",
94  default=True,
95  )
96  splitSubfilters = pexConfig.Field(
97  dtype=bool,
98  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
99  "Instead of at the midpoint",
100  default=False,
101  )
102  regularizeModelIterations = pexConfig.Field(
103  dtype=float,
104  doc="Maximum relative change of the model allowed between iterations."
105  "Set to zero to disable.",
106  default=2.,
107  )
108  regularizeModelFrequency = pexConfig.Field(
109  dtype=float,
110  doc="Maximum relative change of the model allowed between subfilters."
111  "Set to zero to disable.",
112  default=4.,
113  )
114  convergenceMaskPlanes = pexConfig.ListField(
115  dtype=str,
116  default=["DETECTED"],
117  doc="Mask planes to use to calculate convergence."
118  )
119  regularizationWidth = pexConfig.Field(
120  dtype=int,
121  default=2,
122  doc="Minimum radius of a region to include in regularization, in pixels."
123  )
124  imageWarpMethod = pexConfig.Field(
125  dtype=str,
126  doc="Name of the warping kernel to use for shifting the image and variance planes.",
127  default="lanczos3",
128  )
129  maskWarpMethod = pexConfig.Field(
130  dtype=str,
131  doc="Name of the warping kernel to use for shifting the mask plane.",
132  default="bilinear",
133  )
134 
135  def setDefaults(self):
136  CompareWarpAssembleCoaddConfig.setDefaults(self)
137  self.doNImage = True
138  self.warpType = 'direct'
139  self.assembleStaticSkyModel.warpType = self.warpType
140  self.assembleStaticSkyModel.doNImage = self.doNImage
141  self.statistic = 'MEAN'
142 
143 
145  """Assemble DCR coadded images from a set of warps.
146 
147  Attributes
148  ----------
149  bufferSize : `int`
150  The number of pixels to grow each subregion by to allow for DCR.
151  warpCtrl : `lsst.afw.math.WarpingControl`
152  Configuration settings for warping an image
153 
154  Notes
155  -----
156  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
157  Warps (also called coadded temporary exposures), including the effects of
158  Differential Chromatic Refraction (DCR).
159  For full details of the mathematics and algorithm, please see
160  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
161 
162  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
163  each subfilter used in the iterative calculation.
164  It begins by dividing the bandpass-defining filter into N equal bandwidth
165  "subfilters", and divides the flux in each pixel from an initial coadd
166  equally into each as a "dcrModel". Because the airmass and parallactic
167  angle of each individual exposure is known, we can calculate the shift
168  relative to the center of the band in each subfilter due to DCR. For each
169  exposure we apply this shift as a linear transformation to the dcrModels
170  and stack the results to produce a DCR-matched exposure. The matched
171  exposures are subtracted from the input exposures to produce a set of
172  residual images, and these residuals are reverse shifted for each
173  exposures' subfilters and stacked. The shifted and stacked residuals are
174  added to the dcrModels to produce a new estimate of the flux in each pixel
175  within each subfilter. The dcrModels are solved for iteratively, which
176  continues until the solution from a new iteration improves by less than
177  a set percentage, or a maximum number of iterations is reached.
178  Two forms of regularization are employed to reduce unphysical results.
179  First, the new solution is averaged with the solution from the previous
180  iteration, which mitigates oscillating solutions where the model
181  overshoots with alternating very high and low values.
182  Second, a common degeneracy when the data have a limited range of airmass or
183  parallactic angle values is for one subfilter to be fit with very low or
184  negative values, while another subfilter is fit with very high values. This
185  typically appears in the form of holes next to sources in one subfilter,
186  and corresponding extended wings in another. Because each subfilter has
187  a narrow bandwidth we assume that physical sources that are above the noise
188  level will not vary in flux by more than a factor of `frequencyClampFactor`
189  between subfilters, and pixels that have flux deviations larger than that
190  factor will have the excess flux distributed evenly among all subfilters.
191  """
192 
193  ConfigClass = DcrAssembleCoaddConfig
194  _DefaultName = "dcrAssembleCoadd"
195 
196  @pipeBase.timeMethod
197  def runDataRef(self, dataRef, selectDataList=[]):
198  """Assemble a coadd from a set of warps.
199 
200  Coadd a set of Warps. Compute weights to be applied to each Warp and
201  find scalings to match the photometric zeropoint to a reference Warp.
202  Assemble the Warps using run method.
203  Forward model chromatic effects across multiple subfilters,
204  and subtract from the input Warps to build sets of residuals.
205  Use the residuals to construct a new ``DcrModel`` for each subfilter,
206  and iterate until the model converges.
207  Interpolate over NaNs and optionally write the coadd to disk.
208  Return the coadded exposure.
209 
210  Parameters
211  ----------
212  dataRef : `lsst.daf.persistence.ButlerDataRef`
213  Data reference defining the patch for coaddition and the
214  reference Warp
215  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
216  List of data references to warps. Data to be coadded will be
217  selected from this list based on overlap with the patch defined by
218  the data reference.
219 
220  Returns
221  -------
222  results : `lsst.pipe.base.Struct`
223  The Struct contains the following fields:
224 
225  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
226  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
227  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
228  - ``dcrNImages``: `list` of exposure count images for each subfilter
229  """
230  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
231  for subfilter in range(self.config.dcrNumSubfilters):
232  self.processResults(results.dcrCoadds[subfilter], dataRef)
233  if self.config.doWrite:
234  self.log.info("Persisting dcrCoadd")
235  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
236  numSubfilters=self.config.dcrNumSubfilters)
237  if self.config.doNImage and results.dcrNImages is not None:
238  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
239  numSubfilters=self.config.dcrNumSubfilters)
240 
241  return results
242 
243  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
244  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
245 
246  Sets the properties ``warpCtrl`` and ``bufferSize``.
247 
248  Parameters
249  ----------
250  templateCoadd : `lsst.afw.image.ExposureF`
251  The initial coadd exposure before accounting for DCR.
252  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
253  The data references to the input warped exposures.
254  weightList : `list` of `float`
255  The weight to give each input exposure in the coadd
256  Will be modified in place if ``doAirmassWeight`` is set.
257 
258  Returns
259  -------
260  dcrModels : `lsst.pipe.tasks.DcrModel`
261  Best fit model of the true sky after correcting chromatic effects.
262 
263  Raises
264  ------
265  NotImplementedError
266  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
267  """
268  filterInfo = templateCoadd.getFilter()
269  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
270  raise NotImplementedError("No minimum/maximum wavelength information found"
271  " in the filter definition! Please add lambdaMin and lambdaMax"
272  " to the Mapper class in your obs package.")
273  tempExpName = self.getTempExpDatasetName(self.warpType)
274  dcrShifts = []
275  for visitNum, tempExpRef in enumerate(tempExpRefList):
276  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
277  airmass = visitInfo.getBoresightAirmass()
278  if self.config.doAirmassWeight:
279  weightList[visitNum] *= airmass
280  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
281  filterInfo, self.config.dcrNumSubfilters))))
282  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
283  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
284  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
285  warpCache = 0
286  warpInterpLength = max(self.config.subregionSize)
287  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
288  self.config.maskWarpMethod,
289  cacheSize=warpCache, interpLength=warpInterpLength)
290  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
291  self.config.dcrNumSubfilters,
292  filterInfo=filterInfo,
293  psf=templateCoadd.getPsf())
294  return dcrModels
295 
296  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
297  supplementaryData=None):
298  """Assemble the coadd.
299 
300  Requires additional inputs Struct ``supplementaryData`` to contain a
301  ``templateCoadd`` that serves as the model of the static sky.
302 
303  Find artifacts and apply them to the warps' masks creating a list of
304  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
305  Then pass these alternative masks to the base class's assemble method.
306 
307  Divide the ``templateCoadd`` evenly between each subfilter of a
308  ``DcrModel`` as the starting best estimate of the true wavelength-
309  dependent sky. Forward model the ``DcrModel`` using the known
310  chromatic effects in each subfilter and calculate a convergence metric
311  based on how well the modeled template matches the input warps. If
312  the convergence has not yet reached the desired threshold, then shift
313  and stack the residual images to build a new ``DcrModel``. Apply
314  conditioning to prevent oscillating solutions between iterations or
315  between subfilters.
316 
317  Once the ``DcrModel`` reaches convergence or the maximum number of
318  iterations has been reached, fill the metadata for each subfilter
319  image and make them proper ``coaddExposure``s.
320 
321  Parameters
322  ----------
323  skyInfo : `lsst.pipe.base.Struct`
324  Patch geometry information, from getSkyInfo
325  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
326  The data references to the input warped exposures.
327  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
328  The image scalars correct for the zero point of the exposures.
329  weightList : `list` of `float`
330  The weight to give each input exposure in the coadd
331  supplementaryData : `lsst.pipe.base.Struct`
332  Result struct returned by ``makeSupplementaryData`` with components:
333 
334  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
335 
336  Returns
337  -------
338  result : `lsst.pipe.base.Struct`
339  Result struct with components:
340 
341  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
342  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
343  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
344  - ``dcrNImages``: `list` of exposure count images for each subfilter
345  """
346  templateCoadd = supplementaryData.templateCoadd
347  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
348  badMaskPlanes = self.config.badMaskPlanes[:]
349  badMaskPlanes.append("CLIPPED")
350  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
351  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
352  # Psf-Matching moves the real edge inwards
353  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
354 
355  stats = self.prepareStats(mask=badPixelMask)
356  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
357  if self.config.doNImage:
358  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
359  tempExpRefList, spanSetMaskList, stats.ctrl)
360  nImage = afwImage.ImageU(skyInfo.bbox)
361  # Note that this nImage will be a factor of dcrNumSubfilters higher than
362  # the nImage returned by assembleCoadd for most pixels. This is because each
363  # subfilter may have a different nImage, and fractional values are not allowed.
364  for dcrNImage in dcrNImages:
365  nImage += dcrNImage
366  else:
367  dcrNImages = None
368 
369  baseMask = templateCoadd.mask
370  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
371  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
372  modelIter = 0
373  self.log.info("Computing coadd over %s", subBBox)
374  dcrBBox = afwGeom.Box2I(subBBox)
375  dcrBBox.grow(self.bufferSize)
376  dcrBBox.clip(dcrModels.bbox)
377  if self.config.useModelWeights:
378  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
379  else:
380  modelWeights = 1.
381  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
382  imageScalerList, weightList, spanSetMaskList,
383  stats.ctrl)
384  self.log.info("Initial convergence : %s", convergenceMetric)
385  convergenceList = [convergenceMetric]
386  gainList = []
387  convergenceCheck = 1.
388  subfilterVariance = None
389  while (convergenceCheck > self.config.convergenceThreshold or
390  modelIter < self.config.minNumIter):
391  gain = self.calculateGain(convergenceList, gainList)
392  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
393  weightList, spanSetMaskList, stats.flags, stats.ctrl,
394  convergenceMetric, baseMask, subfilterVariance, gain,
395  modelWeights)
396  if self.config.useConvergence:
397  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
398  imageScalerList, weightList,
399  spanSetMaskList,
400  stats.ctrl)
401  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
402  if convergenceCheck < 0:
403  self.log.warn("Coadd %s diverged before reaching maximum iterations or"
404  " desired convergence improvement of %s."
405  " Divergence: %s",
406  subBBox, self.config.convergenceThreshold, convergenceCheck)
407  break
408  convergenceList.append(convergenceMetric)
409  if modelIter > self.config.maxNumIter:
410  if self.config.useConvergence:
411  self.log.warn("Coadd %s reached maximum iterations before reaching"
412  " desired convergence improvement of %s."
413  " Final convergence improvement: %s",
414  subBBox, self.config.convergenceThreshold, convergenceCheck)
415  break
416 
417  if self.config.useConvergence:
418  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
419  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
420  modelIter += 1
421  else:
422  if self.config.useConvergence:
423  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
424  subBBox, convergenceMetric, modelIter)
425  else:
426  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
427  if self.config.useConvergence:
428  self.log.info("Final convergence improvement was %.4f%% overall",
429  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
430 
431  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
432  calibration=self.scaleZeroPoint.getCalib(),
433  coaddInputs=self.inputRecorder.makeCoaddInputs(),
434  mask=templateCoadd.mask)
435  coaddExposure = self.stackCoadd(dcrCoadds)
436  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
437  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
438 
439  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
440  """Calculate the number of exposures contributing to each subfilter.
441 
442  Parameters
443  ----------
444  dcrModels : `lsst.pipe.tasks.DcrModel`
445  Best fit model of the true sky after correcting chromatic effects.
446  bbox : `lsst.afw.geom.box.Box2I`
447  Bounding box of the patch to coadd.
448  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
449  The data references to the input warped exposures.
450  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
451  Each element is dict with keys = mask plane name to add the spans to
452  statsCtrl : `lsst.afw.math.StatisticsControl`
453  Statistics control object for coadd
454 
455  Returns
456  -------
457  dcrNImages : `list` of `lsst.afw.image.ImageU`
458  List of exposure count images for each subfilter
459  """
460  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
461  tempExpName = self.getTempExpDatasetName(self.warpType)
462  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
463  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
464  visitInfo = exposure.getInfo().getVisitInfo()
465  wcs = exposure.getInfo().getWcs()
466  mask = exposure.mask
467  if altMaskSpans is not None:
468  self.applyAltMaskPlanes(mask, altMaskSpans)
469  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
470  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
471  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
472  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
473  return dcrNImages
474 
475  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
476  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
477  baseMask, subfilterVariance, gain, modelWeights):
478  """Assemble the DCR coadd for a sub-region.
479 
480  Build a DCR-matched template for each input exposure, then shift the
481  residuals according to the DCR in each subfilter.
482  Stack the shifted residuals and apply them as a correction to the
483  solution from the previous iteration.
484  Restrict the new model solutions from varying by more than a factor of
485  `modelClampFactor` from the last solution, and additionally restrict the
486  individual subfilter models from varying by more than a factor of
487  `frequencyClampFactor` from their average.
488  Finally, mitigate potentially oscillating solutions by averaging the new
489  solution with the solution from the previous iteration, weighted by
490  their convergence metric.
491 
492  Parameters
493  ----------
494  dcrModels : `lsst.pipe.tasks.DcrModel`
495  Best fit model of the true sky after correcting chromatic effects.
496  bbox : `lsst.afw.geom.box.Box2I`
497  Bounding box of the subregion to coadd.
498  dcrBBox :`lsst.afw.geom.box.Box2I`
499  Sub-region of the coadd which includes a buffer to allow for DCR.
500  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
501  The data references to the input warped exposures.
502  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
503  The image scalars correct for the zero point of the exposures.
504  weightList : `list` of `float`
505  The weight to give each input exposure in the coadd
506  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
507  Each element is dict with keys = mask plane name to add the spans to
508  statsFlags : `lsst.afw.math.Property`
509  Statistics settings for coaddition.
510  statsCtrl : `lsst.afw.math.StatisticsControl`
511  Statistics control object for coadd
512  convergenceMetric : `float`
513  Quality of fit metric for the matched templates of the input images.
514  baseMask : `lsst.afw.image.Mask`
515  Mask of the initial template coadd.
516  subfilterVariance : `list` of `numpy.ndarray`
517  The variance of each coadded subfilter image.
518  gain : `float`, optional
519  Relative weight to give the new solution when updating the model.
520  modelWeights : `numpy.ndarray` or `float`
521  A 2D array of weight values that tapers smoothly to zero away from detected sources.
522  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
523  """
524  tempExpName = self.getTempExpDatasetName(self.warpType)
525  residualGeneratorList = []
526 
527  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
528  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
529  visitInfo = exposure.getInfo().getVisitInfo()
530  wcs = exposure.getInfo().getWcs()
531  maskedImage = exposure.maskedImage
532  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
533  bbox=dcrBBox, wcs=wcs, mask=baseMask,
534  splitSubfilters=self.config.splitSubfilters)
535  imageScaler.scaleMaskedImage(maskedImage)
536  if altMaskSpans is not None:
537  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
538 
539  if self.config.removeMaskPlanes:
540  self.removeMaskPlanes(maskedImage)
541  maskedImage -= templateImage
542  maskedImage.image.array *= modelWeights
543  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
544  dcrModels.filter))
545 
546  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
547  statsFlags, statsCtrl, weightList,
548  mask=baseMask, gain=gain)
549  dcrModels.assign(dcrSubModelOut, bbox)
550 
551  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
552  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
553 
554  Parameters
555  ----------
556  residual : `lsst.afw.image.MaskedImageF`
557  The residual masked image for one exposure,
558  after subtracting the matched template
559  visitInfo : `lsst.afw.image.VisitInfo`
560  Metadata for the exposure.
561  bbox : `lsst.afw.geom.box.Box2I`
562  Sub-region of the coadd
563  wcs : `lsst.afw.geom.SkyWcs`
564  Coordinate system definition (wcs) for the exposure.
565  filterInfo : `lsst.afw.image.Filter`
566  The filter definition, set in the current instruments' obs package.
567  Required for any calculation of DCR, including making matched templates.
568 
569  Yields
570  ------
571  residualImage : `lsst.afw.image.maskedImageF`
572  The residual image for the next subfilter, shifted for DCR.
573  """
574  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
575  for dcr in dcrShift:
576  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
577 
578  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
579  statsFlags, statsCtrl, weightList,
580  mask, gain):
581  """Calculate a new DcrModel from a set of image residuals.
582 
583  Parameters
584  ----------
585  dcrModels : `lsst.pipe.tasks.DcrModel`
586  Current model of the true sky after correcting chromatic effects.
587  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
588  The residual image for the next subfilter, shifted for DCR.
589  bbox : `lsst.afw.geom.box.Box2I`
590  Sub-region of the coadd
591  statsFlags : `lsst.afw.math.Property`
592  Statistics settings for coaddition.
593  statsCtrl : `lsst.afw.math.StatisticsControl`
594  Statistics control object for coadd
595  weightList : `list` of `float`
596  The weight to give each input exposure in the coadd
597  mask : `lsst.afw.image.Mask`
598  Mask to use for each new model image.
599  gain : `float`
600  Relative weight to give the new solution when updating the model.
601 
602  Returns
603  -------
604  dcrModel : `lsst.pipe.tasks.DcrModel`
605  New model of the true sky after correcting chromatic effects.
606  """
607  maskMap = self.setRejectedMaskMapping(statsCtrl)
608  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
609  newModelImages = []
610  for subfilter, model in enumerate(dcrModels):
611  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
612  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
613  clipped, maskMap)
614  residual.setXY0(bbox.getBegin())
615  # `MaskedImage`s only support in-place addition, so rename for readability
616  residual += model[bbox]
617  newModel = residual
618  # Catch any invalid values
619  badPixels = ~np.isfinite(newModel.image.array)
620  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
621  # every iteration, masked regions will continually expand.
622  newModel.setMask(mask[bbox])
623  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
624  if self.config.regularizeModelIterations > 0:
625  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
626  self.config.regularizeModelIterations,
627  self.config.regularizationWidth)
628  newModelImages.append(newModel)
629  if self.config.regularizeModelFrequency > 0:
630  dcrModels.regularizeModelFreq(newModelImages, bbox,
631  self.config.regularizeModelFrequency,
632  self.config.regularizationWidth)
633  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
634  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
635 
636  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
637  weightList, spanSetMaskList, statsCtrl):
638  """Calculate a quality of fit metric for the matched templates.
639 
640  Parameters
641  ----------
642  dcrModels : `lsst.pipe.tasks.DcrModel`
643  Best fit model of the true sky after correcting chromatic effects.
644  bbox : `lsst.afw.geom.box.Box2I`
645  Sub-region to coadd
646  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
647  The data references to the input warped exposures.
648  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
649  The image scalars correct for the zero point of the exposures.
650  weightList : `list` of `float`
651  The weight to give each input exposure in the coadd
652  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
653  Each element is dict with keys = mask plane name to add the spans to
654  statsCtrl : `lsst.afw.math.StatisticsControl`
655  Statistics control object for coadd
656 
657  Returns
658  -------
659  convergenceMetric : `float`
660  Quality of fit metric for all input exposures, within the sub-region
661  """
662  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
663  nSigma = 3.
664  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
665  bufferSize=self.bufferSize)
666  tempExpName = self.getTempExpDatasetName(self.warpType)
667  weight = 0
668  metric = 0.
669  metricList = {}
670  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
671  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
672  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
673  imageScaler.scaleMaskedImage(exposure.maskedImage)
674  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
675  altMaskSpans=altMaskSpans)
676  metric += singleMetric*expWeight
677  metricList[tempExpRef.dataId["visit"]] = singleMetric
678  weight += expWeight
679  self.log.info("Individual metrics:\n%s", metricList)
680  return 1.0 if weight == 0.0 else metric/weight
681 
682  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
683  statsCtrl, altMaskSpans=None):
684  """Calculate a quality of fit metric for a single matched template.
685 
686  Parameters
687  ----------
688  dcrModels : `lsst.pipe.tasks.DcrModel`
689  Best fit model of the true sky after correcting chromatic effects.
690  exposure : `lsst.afw.image.ExposureF`
691  The input warped exposure to evaluate.
692  significanceImage : `numpy.ndarray`
693  Array of weights for each pixel corresponding to its significance
694  for the convergence calculation.
695  statsCtrl : `lsst.afw.math.StatisticsControl`
696  Statistics control object for coadd
697  altMaskSpans : `dict` containing spanSet lists, or None
698  The keys of the `dict` equal the mask plane name to add the spans to
699 
700  Returns
701  -------
702  convergenceMetric : `float`
703  Quality of fit metric for one exposure, within the sub-region.
704  """
705  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
706  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
707  visitInfo=exposure.getInfo().getVisitInfo(),
708  bbox=exposure.getBBox(),
709  wcs=exposure.getInfo().getWcs())
710  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
711  refVals = np.abs(templateImage.image.array)*significanceImage
712 
713  finitePixels = np.isfinite(diffVals)
714  if altMaskSpans is not None:
715  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
716  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
717  convergeMaskPixels = exposure.mask.array & convergeMask > 0
718  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
719  if np.sum(usePixels) == 0:
720  metric = 0.
721  else:
722  diffUse = diffVals[usePixels]
723  refUse = refVals[usePixels]
724  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
725  return metric
726 
727  def stackCoadd(self, dcrCoadds):
728  """Add a list of sub-band coadds together.
729 
730  Parameters
731  ----------
732  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
733  A list of coadd exposures, each exposure containing
734  the model for one subfilter.
735 
736  Returns
737  -------
738  coaddExposure : `lsst.afw.image.ExposureF`
739  A single coadd exposure that is the sum of the sub-bands.
740  """
741  coaddExposure = dcrCoadds[0].clone()
742  for coadd in dcrCoadds[1:]:
743  coaddExposure.maskedImage += coadd.maskedImage
744  return coaddExposure
745 
746  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
747  mask=None):
748  """Create a list of coadd exposures from a list of masked images.
749 
750  Parameters
751  ----------
752  dcrModels : `lsst.pipe.tasks.DcrModel`
753  Best fit model of the true sky after correcting chromatic effects.
754  skyInfo : `lsst.pipe.base.Struct`
755  Patch geometry information, from getSkyInfo
756  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
757  The data references to the input warped exposures.
758  weightList : `list` of `float`
759  The weight to give each input exposure in the coadd
760  calibration : `lsst.afw.Image.Calib`, optional
761  Scale factor to set the photometric zero point of an exposure.
762  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
763  A record of the observations that are included in the coadd.
764  mask : `lsst.afw.image.Mask`, optional
765  Optional mask to override the values in the final coadd.
766 
767  Returns
768  -------
769  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
770  A list of coadd exposures, each exposure containing
771  the model for one subfilter.
772  """
773  dcrCoadds = []
774  for model in dcrModels:
775  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
776  if calibration is not None:
777  coaddExposure.setCalib(calibration)
778  if coaddInputs is not None:
779  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
780  # Set the metadata for the coadd, including PSF and aperture corrections.
781  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
782  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
783  coaddExposure.setMaskedImage(model[skyInfo.bbox])
784  if mask is not None:
785  coaddExposure.setMask(mask)
786  dcrCoadds.append(coaddExposure)
787  return dcrCoadds
788 
789  def calculateGain(self, convergenceList, gainList):
790  """Calculate the gain to use for the current iteration.
791 
792  After calculating a new DcrModel, each value is averaged with the
793  value in the corresponding pixel from the previous iteration. This
794  reduces oscillating solutions that iterative techniques are plagued by,
795  and speeds convergence. By far the biggest changes to the model
796  happen in the first couple iterations, so we can also use a more
797  aggressive gain later when the model is changing slowly.
798 
799  Parameters
800  ----------
801  convergenceList : `list` of `float`
802  The quality of fit metric from each previous iteration.
803  gainList : `list` of `float`
804  The gains used in each previous iteration: appended with the new
805  gain value.
806  Gains are numbers between ``self.config.baseGain`` and 1.
807 
808  Returns
809  -------
810  gain : `float`
811  Relative weight to give the new solution when updating the model.
812  A value of 1.0 gives equal weight to both solutions.
813 
814  Raises
815  ------
816  ValueError
817  If ``len(convergenceList) != len(gainList)+1``.
818  """
819  nIter = len(convergenceList)
820  if nIter != len(gainList) + 1:
821  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
822  % (len(convergenceList), len(gainList)))
823 
824  if self.config.baseGain is None:
825  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
826  # The more subfilters being modeled, the lower the gain should be.
827  baseGain = 1./(self.config.dcrNumSubfilters - 1)
828  else:
829  baseGain = self.config.baseGain
830 
831  if self.config.useProgressiveGain and nIter > 2:
832  # To calculate the best gain to use, compare the past gains that have been used
833  # with the resulting convergences to estimate the best gain to use.
834  # Algorithmically, this is a Kalman filter.
835  # If forward modeling proceeds perfectly, the convergence metric should
836  # asymptotically approach a final value.
837  # We can estimate that value from the measured changes in convergence
838  # weighted by the gains used in each previous iteration.
839  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
840  for i in range(nIter - 1)]
841  # The convergence metric is strictly positive, so if the estimated final convergence is
842  # less than zero, force it to zero.
843  estFinalConv = np.array(estFinalConv)
844  estFinalConv[estFinalConv < 0] = 0
845  # Because the estimate may slowly change over time, only use the most recent measurements.
846  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
847  lastGain = gainList[-1]
848  lastConv = convergenceList[-2]
849  newConv = convergenceList[-1]
850  # The predicted convergence is the value we would get if the new model calculated
851  # in the previous iteration was perfect. Recall that the updated model that is
852  # actually used is the gain-weighted average of the new and old model,
853  # so the convergence would be similarly weighted.
854  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
855  # If the measured and predicted convergence are very close, that indicates
856  # that our forward model is accurate and we can use a more aggressive gain
857  # If the measured convergence is significantly worse (or better!) than predicted,
858  # that indicates that the model is not converging as expected and
859  # we should use a more conservative gain.
860  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
861  newGain = 1 - abs(delta)
862  # Average the gains to prevent oscillating solutions.
863  newGain = (newGain + lastGain)/2.
864  gain = max(baseGain, newGain)
865  else:
866  gain = baseGain
867  gainList.append(gain)
868  return gain
869 
870  def calculateModelWeights(self, dcrModels, dcrBBox):
871  """Build an array that smoothly tapers to 0 away from detected sources.
872 
873  Parameters
874  ----------
875  dcrModels : `lsst.pipe.tasks.DcrModel`
876  Best fit model of the true sky after correcting chromatic effects.
877  dcrBBox : `lsst.afw.geom.box.Box2I`
878  Sub-region of the coadd which includes a buffer to allow for DCR.
879 
880  Returns
881  -------
882  weights : `numpy.ndarray` or `float`
883  A 2D array of weight values that tapers smoothly to zero away from detected sources.
884  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
885 
886  Raises
887  ------
888  ValueError
889  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
890  """
891  if self.config.modelWeightsWidth < 0:
892  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
893  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
894  convergeMaskPixels = dcrModels.mask.array & convergeMask > 0
895  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
896  weights[convergeMaskPixels] = 1.
897  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
898  weights /= np.max(weights)
899  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:159
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)