lsst.pipe.tasks  16.0-36-g07840cb1
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from scipy import ndimage
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  doc="Maximum number of iterations of forward modeling.",
46  default=8,
47  )
48  minNumIter = pexConfig.Field(
49  dtype=int,
50  doc="Minimum number of iterations of forward modeling.",
51  default=3,
52  )
53  convergenceThreshold = pexConfig.Field(
54  dtype=float,
55  doc="Target relative change in convergence between iterations of forward modeling.",
56  default=0.001,
57  )
58  useConvergence = pexConfig.Field(
59  dtype=bool,
60  doc="Use convergence test as a forward modeling end condition?"
61  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
62  default=True,
63  )
64  baseGain = pexConfig.Field(
65  dtype=float,
66  optional=True,
67  doc="Relative weight to give the new solution vs. the last solution when updating the model."
68  "A value of 1.0 gives equal weight to both solutions."
69  "Small values imply slower convergence of the solution, but can "
70  "help prevent overshooting and failures in the fit."
71  "If ``baseGain`` is None, a conservative gain "
72  "will be calculated from the number of subfilters. ",
73  default=None,
74  )
75  useProgressiveGain = pexConfig.Field(
76  dtype=bool,
77  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
78  "When calculating the next gain, we use up to 5 previous gains and convergence values."
79  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
80  default=True,
81  )
82  doAirmassWeight = pexConfig.Field(
83  dtype=bool,
84  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
85  default=True,
86  )
87  modelWeightsWidth = pexConfig.Field(
88  dtype=float,
89  doc="Width of the region around detected sources to include in the DcrModel.",
90  default=3,
91  )
92  useModelWeights = pexConfig.Field(
93  dtype=bool,
94  doc="Width of the region around detected sources to include in the DcrModel.",
95  default=True,
96  )
97  splitSubfilters = pexConfig.Field(
98  dtype=bool,
99  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
100  "Instead of at the midpoint",
101  default=False,
102  )
103  regularizeModelIterations = pexConfig.Field(
104  dtype=float,
105  doc="Maximum relative change of the model allowed between iterations."
106  "Set to zero to disable.",
107  default=2.,
108  )
109  regularizeModelFrequency = pexConfig.Field(
110  dtype=float,
111  doc="Maximum relative change of the model allowed between subfilters."
112  "Set to zero to disable.",
113  default=4.,
114  )
115  convergenceMaskPlanes = pexConfig.ListField(
116  dtype=str,
117  default=["DETECTED"],
118  doc="Mask planes to use to calculate convergence."
119  )
120  regularizationWidth = pexConfig.Field(
121  dtype=int,
122  default=2,
123  doc="Minimum radius of a region to include in regularization, in pixels."
124  )
125  imageWarpMethod = pexConfig.Field(
126  dtype=str,
127  doc="Name of the warping kernel to use for shifting the image and variance planes.",
128  default="lanczos3",
129  )
130  maskWarpMethod = pexConfig.Field(
131  dtype=str,
132  doc="Name of the warping kernel to use for shifting the mask plane.",
133  default="bilinear",
134  )
135 
136  def setDefaults(self):
137  CompareWarpAssembleCoaddConfig.setDefaults(self)
138  self.doNImage = True
139  self.warpType = 'direct'
140  self.assembleStaticSkyModel.warpType = self.warpType
141  self.assembleStaticSkyModel.doNImage = self.doNImage
142  self.statistic = 'MEAN'
143 
144 
146  """Assemble DCR coadded images from a set of warps.
147 
148  Attributes
149  ----------
150  bufferSize : `int`
151  The number of pixels to grow each subregion by to allow for DCR.
152  warpCtrl : `lsst.afw.math.WarpingControl`
153  Configuration settings for warping an image
154 
155  Notes
156  -----
157  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
158  Warps (also called coadded temporary exposures), including the effects of
159  Differential Chromatic Refraction (DCR).
160  For full details of the mathematics and algorithm, please see
161  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
162 
163  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
164  each subfilter used in the iterative calculation.
165  It begins by dividing the bandpass-defining filter into N equal bandwidth
166  "subfilters", and divides the flux in each pixel from an initial coadd
167  equally into each as a "dcrModel". Because the airmass and parallactic
168  angle of each individual exposure is known, we can calculate the shift
169  relative to the center of the band in each subfilter due to DCR. For each
170  exposure we apply this shift as a linear transformation to the dcrModels
171  and stack the results to produce a DCR-matched exposure. The matched
172  exposures are subtracted from the input exposures to produce a set of
173  residual images, and these residuals are reverse shifted for each
174  exposures' subfilters and stacked. The shifted and stacked residuals are
175  added to the dcrModels to produce a new estimate of the flux in each pixel
176  within each subfilter. The dcrModels are solved for iteratively, which
177  continues until the solution from a new iteration improves by less than
178  a set percentage, or a maximum number of iterations is reached.
179  Two forms of regularization are employed to reduce unphysical results.
180  First, the new solution is averaged with the solution from the previous
181  iteration, which mitigates oscillating solutions where the model
182  overshoots with alternating very high and low values.
183  Second, a common degeneracy when the data have a limited range of airmass or
184  parallactic angle values is for one subfilter to be fit with very low or
185  negative values, while another subfilter is fit with very high values. This
186  typically appears in the form of holes next to sources in one subfilter,
187  and corresponding extended wings in another. Because each subfilter has
188  a narrow bandwidth we assume that physical sources that are above the noise
189  level will not vary in flux by more than a factor of `frequencyClampFactor`
190  between subfilters, and pixels that have flux deviations larger than that
191  factor will have the excess flux distributed evenly among all subfilters.
192  """
193 
194  ConfigClass = DcrAssembleCoaddConfig
195  _DefaultName = "dcrAssembleCoadd"
196 
197  @pipeBase.timeMethod
198  def runDataRef(self, dataRef, selectDataList=[]):
199  """Assemble a coadd from a set of warps.
200 
201  Coadd a set of Warps. Compute weights to be applied to each Warp and
202  find scalings to match the photometric zeropoint to a reference Warp.
203  Assemble the Warps using run method.
204  Forward model chromatic effects across multiple subfilters,
205  and subtract from the input Warps to build sets of residuals.
206  Use the residuals to construct a new ``DcrModel`` for each subfilter,
207  and iterate until the model converges.
208  Interpolate over NaNs and optionally write the coadd to disk.
209  Return the coadded exposure.
210 
211  Parameters
212  ----------
213  dataRef : `lsst.daf.persistence.ButlerDataRef`
214  Data reference defining the patch for coaddition and the
215  reference Warp
216  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
217  List of data references to warps. Data to be coadded will be
218  selected from this list based on overlap with the patch defined by
219  the data reference.
220 
221  Returns
222  -------
223  results : `lsst.pipe.base.Struct`
224  The Struct contains the following fields:
225 
226  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
227  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
228  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
229  - ``dcrNImages``: `list` of exposure count images for each subfilter
230  """
231  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
232  for subfilter in range(self.config.dcrNumSubfilters):
233  self.processResults(results.dcrCoadds[subfilter], dataRef)
234  if self.config.doWrite:
235  self.log.info("Persisting dcrCoadd")
236  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
237  numSubfilters=self.config.dcrNumSubfilters)
238  if self.config.doNImage and results.dcrNImages is not None:
239  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
240  numSubfilters=self.config.dcrNumSubfilters)
241 
242  return results
243 
244  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
245  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
246 
247  Sets the properties ``warpCtrl`` and ``bufferSize``.
248 
249  Parameters
250  ----------
251  templateCoadd : `lsst.afw.image.ExposureF`
252  The initial coadd exposure before accounting for DCR.
253  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
254  The data references to the input warped exposures.
255  weightList : `list` of `float`
256  The weight to give each input exposure in the coadd
257  Will be modified in place if ``doAirmassWeight`` is set.
258 
259  Returns
260  -------
261  dcrModels : `lsst.pipe.tasks.DcrModel`
262  Best fit model of the true sky after correcting chromatic effects.
263 
264  Raises
265  ------
266  NotImplementedError
267  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
268  """
269  filterInfo = templateCoadd.getFilter()
270  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
271  raise NotImplementedError("No minimum/maximum wavelength information found"
272  " in the filter definition! Please add lambdaMin and lambdaMax"
273  " to the Mapper class in your obs package.")
274  tempExpName = self.getTempExpDatasetName(self.warpType)
275  dcrShifts = []
276  for visitNum, tempExpRef in enumerate(tempExpRefList):
277  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
278  airmass = visitInfo.getBoresightAirmass()
279  if self.config.doAirmassWeight:
280  weightList[visitNum] *= airmass
281  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
282  filterInfo, self.config.dcrNumSubfilters))))
283  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
284  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
285  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
286  warpCache = 0
287  warpInterpLength = max(self.config.subregionSize)
288  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
289  self.config.maskWarpMethod,
290  cacheSize=warpCache, interpLength=warpInterpLength)
291  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
292  self.config.dcrNumSubfilters,
293  filterInfo=filterInfo,
294  psf=templateCoadd.getPsf())
295  return dcrModels
296 
297  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
298  supplementaryData=None):
299  """Assemble the coadd.
300 
301  Requires additional inputs Struct ``supplementaryData`` to contain a
302  ``templateCoadd`` that serves as the model of the static sky.
303 
304  Find artifacts and apply them to the warps' masks creating a list of
305  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
306  Then pass these alternative masks to the base class's assemble method.
307 
308  Divide the ``templateCoadd`` evenly between each subfilter of a
309  ``DcrModel`` as the starting best estimate of the true wavelength-
310  dependent sky. Forward model the ``DcrModel`` using the known
311  chromatic effects in each subfilter and calculate a convergence metric
312  based on how well the modeled template matches the input warps. If
313  the convergence has not yet reached the desired threshold, then shift
314  and stack the residual images to build a new ``DcrModel``. Apply
315  conditioning to prevent oscillating solutions between iterations or
316  between subfilters.
317 
318  Once the ``DcrModel`` reaches convergence or the maximum number of
319  iterations has been reached, fill the metadata for each subfilter
320  image and make them proper ``coaddExposure``s.
321 
322  Parameters
323  ----------
324  skyInfo : `lsst.pipe.base.Struct`
325  Patch geometry information, from getSkyInfo
326  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
327  The data references to the input warped exposures.
328  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
329  The image scalars correct for the zero point of the exposures.
330  weightList : `list` of `float`
331  The weight to give each input exposure in the coadd
332  supplementaryData : `lsst.pipe.base.Struct`
333  Result struct returned by ``makeSupplementaryData`` with components:
334 
335  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
336 
337  Returns
338  -------
339  result : `lsst.pipe.base.Struct`
340  Result struct with components:
341 
342  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
343  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
344  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
345  - ``dcrNImages``: `list` of exposure count images for each subfilter
346  """
347  templateCoadd = supplementaryData.templateCoadd
348  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
349  badMaskPlanes = self.config.badMaskPlanes[:]
350  badMaskPlanes.append("CLIPPED")
351  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
352  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
353  # Psf-Matching moves the real edge inwards
354  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
355 
356  stats = self.prepareStats(mask=badPixelMask)
357  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
358  if self.config.doNImage:
359  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
360  tempExpRefList, spanSetMaskList, stats.ctrl)
361  nImage = afwImage.ImageU(skyInfo.bbox)
362  # Note that this nImage will be a factor of dcrNumSubfilters higher than
363  # the nImage returned by assembleCoadd for most pixels. This is because each
364  # subfilter may have a different nImage, and fractional values are not allowed.
365  for dcrNImage in dcrNImages:
366  nImage += dcrNImage
367  else:
368  dcrNImages = None
369 
370  baseMask = templateCoadd.mask
371  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
372  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
373  modelIter = 0
374  self.log.info("Computing coadd over %s", subBBox)
375  dcrBBox = afwGeom.Box2I(subBBox)
376  dcrBBox.grow(self.bufferSize)
377  dcrBBox.clip(dcrModels.bbox)
378  if self.config.useModelWeights:
379  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
380  else:
381  modelWeights = 1.
382  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
383  imageScalerList, weightList, spanSetMaskList,
384  stats.ctrl)
385  self.log.info("Initial convergence : %s", convergenceMetric)
386  convergenceList = [convergenceMetric]
387  gainList = []
388  convergenceCheck = 1.
389  subfilterVariance = None
390  while (convergenceCheck > self.config.convergenceThreshold or
391  modelIter < self.config.minNumIter):
392  gain = self.calculateGain(convergenceList, gainList)
393  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
394  weightList, spanSetMaskList, stats.flags, stats.ctrl,
395  convergenceMetric, baseMask, subfilterVariance, gain,
396  modelWeights)
397  if self.config.useConvergence:
398  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
399  imageScalerList, weightList,
400  spanSetMaskList,
401  stats.ctrl)
402  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
403  if convergenceCheck < 0:
404  self.log.warn("Coadd %s diverged before reaching maximum iterations or"
405  " desired convergence improvement of %s."
406  " Divergence: %s",
407  subBBox, self.config.convergenceThreshold, convergenceCheck)
408  break
409  convergenceList.append(convergenceMetric)
410  if modelIter > self.config.maxNumIter:
411  if self.config.useConvergence:
412  self.log.warn("Coadd %s reached maximum iterations before reaching"
413  " desired convergence improvement of %s."
414  " Final convergence improvement: %s",
415  subBBox, self.config.convergenceThreshold, convergenceCheck)
416  break
417 
418  if self.config.useConvergence:
419  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
420  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
421  modelIter += 1
422  else:
423  if self.config.useConvergence:
424  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
425  subBBox, convergenceMetric, modelIter)
426  else:
427  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
428  if self.config.useConvergence:
429  self.log.info("Final convergence improvement was %.4f%% overall",
430  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
431 
432  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
433  calibration=self.scaleZeroPoint.getCalib(),
434  coaddInputs=self.inputRecorder.makeCoaddInputs(),
435  mask=templateCoadd.mask)
436  coaddExposure = self.stackCoadd(dcrCoadds)
437  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
438  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
439 
440  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
441  """Calculate the number of exposures contributing to each subfilter.
442 
443  Parameters
444  ----------
445  dcrModels : `lsst.pipe.tasks.DcrModel`
446  Best fit model of the true sky after correcting chromatic effects.
447  bbox : `lsst.afw.geom.box.Box2I`
448  Bounding box of the patch to coadd.
449  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
450  The data references to the input warped exposures.
451  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
452  Each element is dict with keys = mask plane name to add the spans to
453  statsCtrl : `lsst.afw.math.StatisticsControl`
454  Statistics control object for coadd
455 
456  Returns
457  -------
458  dcrNImages : `list` of `lsst.afw.image.ImageU`
459  List of exposure count images for each subfilter
460  """
461  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
462  tempExpName = self.getTempExpDatasetName(self.warpType)
463  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
464  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
465  visitInfo = exposure.getInfo().getVisitInfo()
466  wcs = exposure.getInfo().getWcs()
467  mask = exposure.mask
468  if altMaskSpans is not None:
469  self.applyAltMaskPlanes(mask, altMaskSpans)
470  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
471  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
472  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
473  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
474  return dcrNImages
475 
476  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
477  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
478  baseMask, subfilterVariance, gain, modelWeights):
479  """Assemble the DCR coadd for a sub-region.
480 
481  Build a DCR-matched template for each input exposure, then shift the
482  residuals according to the DCR in each subfilter.
483  Stack the shifted residuals and apply them as a correction to the
484  solution from the previous iteration.
485  Restrict the new model solutions from varying by more than a factor of
486  `modelClampFactor` from the last solution, and additionally restrict the
487  individual subfilter models from varying by more than a factor of
488  `frequencyClampFactor` from their average.
489  Finally, mitigate potentially oscillating solutions by averaging the new
490  solution with the solution from the previous iteration, weighted by
491  their convergence metric.
492 
493  Parameters
494  ----------
495  dcrModels : `lsst.pipe.tasks.DcrModel`
496  Best fit model of the true sky after correcting chromatic effects.
497  bbox : `lsst.afw.geom.box.Box2I`
498  Bounding box of the subregion to coadd.
499  dcrBBox :`lsst.afw.geom.box.Box2I`
500  Sub-region of the coadd which includes a buffer to allow for DCR.
501  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
502  The data references to the input warped exposures.
503  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
504  The image scalars correct for the zero point of the exposures.
505  weightList : `list` of `float`
506  The weight to give each input exposure in the coadd
507  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
508  Each element is dict with keys = mask plane name to add the spans to
509  statsFlags : `lsst.afw.math.Property`
510  Statistics settings for coaddition.
511  statsCtrl : `lsst.afw.math.StatisticsControl`
512  Statistics control object for coadd
513  convergenceMetric : `float`
514  Quality of fit metric for the matched templates of the input images.
515  baseMask : `lsst.afw.image.Mask`
516  Mask of the initial template coadd.
517  subfilterVariance : `list` of `numpy.ndarray`
518  The variance of each coadded subfilter image.
519  gain : `float`, optional
520  Relative weight to give the new solution when updating the model.
521  modelWeights : `numpy.ndarray` or `float`
522  A 2D array of weight values that tapers smoothly to zero away from detected sources.
523  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
524  """
525  tempExpName = self.getTempExpDatasetName(self.warpType)
526  residualGeneratorList = []
527 
528  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
529  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
530  visitInfo = exposure.getInfo().getVisitInfo()
531  wcs = exposure.getInfo().getWcs()
532  maskedImage = exposure.maskedImage
533  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
534  bbox=dcrBBox, wcs=wcs, mask=baseMask,
535  splitSubfilters=self.config.splitSubfilters)
536  imageScaler.scaleMaskedImage(maskedImage)
537  if altMaskSpans is not None:
538  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
539 
540  if self.config.removeMaskPlanes:
541  self.removeMaskPlanes(maskedImage)
542  maskedImage -= templateImage
543  maskedImage.image.array *= modelWeights
544  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
545  dcrModels.filter))
546 
547  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
548  statsFlags, statsCtrl, weightList,
549  mask=baseMask, gain=gain)
550  dcrModels.assign(dcrSubModelOut, bbox)
551 
552  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
553  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
554 
555  Parameters
556  ----------
557  residual : `lsst.afw.image.MaskedImageF`
558  The residual masked image for one exposure,
559  after subtracting the matched template
560  visitInfo : `lsst.afw.image.VisitInfo`
561  Metadata for the exposure.
562  bbox : `lsst.afw.geom.box.Box2I`
563  Sub-region of the coadd
564  wcs : `lsst.afw.geom.SkyWcs`
565  Coordinate system definition (wcs) for the exposure.
566  filterInfo : `lsst.afw.image.Filter`
567  The filter definition, set in the current instruments' obs package.
568  Required for any calculation of DCR, including making matched templates.
569 
570  Yields
571  ------
572  residualImage : `lsst.afw.image.maskedImageF`
573  The residual image for the next subfilter, shifted for DCR.
574  """
575  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
576  for dcr in dcrShift:
577  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
578 
579  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
580  statsFlags, statsCtrl, weightList,
581  mask, gain):
582  """Calculate a new DcrModel from a set of image residuals.
583 
584  Parameters
585  ----------
586  dcrModels : `lsst.pipe.tasks.DcrModel`
587  Current model of the true sky after correcting chromatic effects.
588  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
589  The residual image for the next subfilter, shifted for DCR.
590  bbox : `lsst.afw.geom.box.Box2I`
591  Sub-region of the coadd
592  statsFlags : `lsst.afw.math.Property`
593  Statistics settings for coaddition.
594  statsCtrl : `lsst.afw.math.StatisticsControl`
595  Statistics control object for coadd
596  weightList : `list` of `float`
597  The weight to give each input exposure in the coadd
598  mask : `lsst.afw.image.Mask`
599  Mask to use for each new model image.
600  gain : `float`
601  Relative weight to give the new solution when updating the model.
602 
603  Returns
604  -------
605  dcrModel : `lsst.pipe.tasks.DcrModel`
606  New model of the true sky after correcting chromatic effects.
607  """
608  maskMap = self.setRejectedMaskMapping(statsCtrl)
609  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
610  newModelImages = []
611  for subfilter, model in enumerate(dcrModels):
612  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
613  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
614  clipped, maskMap)
615  residual.setXY0(bbox.getBegin())
616  # `MaskedImage`s only support in-place addition, so rename for readability
617  residual += model[bbox]
618  newModel = residual
619  # Catch any invalid values
620  badPixels = ~np.isfinite(newModel.image.array)
621  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
622  # every iteration, masked regions will continually expand.
623  newModel.setMask(mask[bbox])
624  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
625  if self.config.regularizeModelIterations > 0:
626  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
627  self.config.regularizeModelIterations,
628  self.config.regularizationWidth)
629  newModelImages.append(newModel)
630  if self.config.regularizeModelFrequency > 0:
631  dcrModels.regularizeModelFreq(newModelImages, bbox,
632  self.config.regularizeModelFrequency,
633  self.config.regularizationWidth)
634  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
635  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
636 
637  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
638  weightList, spanSetMaskList, statsCtrl):
639  """Calculate a quality of fit metric for the matched templates.
640 
641  Parameters
642  ----------
643  dcrModels : `lsst.pipe.tasks.DcrModel`
644  Best fit model of the true sky after correcting chromatic effects.
645  bbox : `lsst.afw.geom.box.Box2I`
646  Sub-region to coadd
647  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
648  The data references to the input warped exposures.
649  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
650  The image scalars correct for the zero point of the exposures.
651  weightList : `list` of `float`
652  The weight to give each input exposure in the coadd
653  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
654  Each element is dict with keys = mask plane name to add the spans to
655  statsCtrl : `lsst.afw.math.StatisticsControl`
656  Statistics control object for coadd
657 
658  Returns
659  -------
660  convergenceMetric : `float`
661  Quality of fit metric for all input exposures, within the sub-region
662  """
663  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
664  nSigma = 3.
665  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
666  bufferSize=self.bufferSize)
667  tempExpName = self.getTempExpDatasetName(self.warpType)
668  weight = 0
669  metric = 0.
670  metricList = {}
671  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
672  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
673  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
674  imageScaler.scaleMaskedImage(exposure.maskedImage)
675  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
676  altMaskSpans=altMaskSpans)
677  metric += singleMetric*expWeight
678  metricList[tempExpRef.dataId["visit"]] = singleMetric
679  weight += expWeight
680  self.log.info("Individual metrics:\n%s", metricList)
681  return 1.0 if weight == 0.0 else metric/weight
682 
683  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
684  statsCtrl, altMaskSpans=None):
685  """Calculate a quality of fit metric for a single matched template.
686 
687  Parameters
688  ----------
689  dcrModels : `lsst.pipe.tasks.DcrModel`
690  Best fit model of the true sky after correcting chromatic effects.
691  exposure : `lsst.afw.image.ExposureF`
692  The input warped exposure to evaluate.
693  significanceImage : `numpy.ndarray`
694  Array of weights for each pixel corresponding to its significance
695  for the convergence calculation.
696  statsCtrl : `lsst.afw.math.StatisticsControl`
697  Statistics control object for coadd
698  altMaskSpans : `dict` containing spanSet lists, or None
699  The keys of the `dict` equal the mask plane name to add the spans to
700 
701  Returns
702  -------
703  convergenceMetric : `float`
704  Quality of fit metric for one exposure, within the sub-region.
705  """
706  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
707  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
708  visitInfo=exposure.getInfo().getVisitInfo(),
709  bbox=exposure.getBBox(),
710  wcs=exposure.getInfo().getWcs())
711  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
712  refVals = np.abs(templateImage.image.array)*significanceImage
713 
714  finitePixels = np.isfinite(diffVals)
715  if altMaskSpans is not None:
716  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
717  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
718  convergeMaskPixels = exposure.mask.array & convergeMask > 0
719  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
720  if np.sum(usePixels) == 0:
721  metric = 0.
722  else:
723  diffUse = diffVals[usePixels]
724  refUse = refVals[usePixels]
725  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
726  return metric
727 
728  def stackCoadd(self, dcrCoadds):
729  """Add a list of sub-band coadds together.
730 
731  Parameters
732  ----------
733  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
734  A list of coadd exposures, each exposure containing
735  the model for one subfilter.
736 
737  Returns
738  -------
739  coaddExposure : `lsst.afw.image.ExposureF`
740  A single coadd exposure that is the sum of the sub-bands.
741  """
742  coaddExposure = dcrCoadds[0].clone()
743  for coadd in dcrCoadds[1:]:
744  coaddExposure.maskedImage += coadd.maskedImage
745  return coaddExposure
746 
747  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
748  mask=None):
749  """Create a list of coadd exposures from a list of masked images.
750 
751  Parameters
752  ----------
753  dcrModels : `lsst.pipe.tasks.DcrModel`
754  Best fit model of the true sky after correcting chromatic effects.
755  skyInfo : `lsst.pipe.base.Struct`
756  Patch geometry information, from getSkyInfo
757  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
758  The data references to the input warped exposures.
759  weightList : `list` of `float`
760  The weight to give each input exposure in the coadd
761  calibration : `lsst.afw.Image.Calib`, optional
762  Scale factor to set the photometric zero point of an exposure.
763  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
764  A record of the observations that are included in the coadd.
765  mask : `lsst.afw.image.Mask`, optional
766  Optional mask to override the values in the final coadd.
767 
768  Returns
769  -------
770  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
771  A list of coadd exposures, each exposure containing
772  the model for one subfilter.
773  """
774  dcrCoadds = []
775  for model in dcrModels:
776  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
777  if calibration is not None:
778  coaddExposure.setCalib(calibration)
779  if coaddInputs is not None:
780  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
781  # Set the metadata for the coadd, including PSF and aperture corrections.
782  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
783  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
784  coaddExposure.setMaskedImage(model[skyInfo.bbox])
785  if mask is not None:
786  coaddExposure.setMask(mask)
787  dcrCoadds.append(coaddExposure)
788  return dcrCoadds
789 
790  def calculateGain(self, convergenceList, gainList):
791  """Calculate the gain to use for the current iteration.
792 
793  After calculating a new DcrModel, each value is averaged with the
794  value in the corresponding pixel from the previous iteration. This
795  reduces oscillating solutions that iterative techniques are plagued by,
796  and speeds convergence. By far the biggest changes to the model
797  happen in the first couple iterations, so we can also use a more
798  aggressive gain later when the model is changing slowly.
799 
800  Parameters
801  ----------
802  convergenceList : `list` of `float`
803  The quality of fit metric from each previous iteration.
804  gainList : `list` of `float`
805  The gains used in each previous iteration: appended with the new
806  gain value.
807  Gains are numbers between ``self.config.baseGain`` and 1.
808 
809  Returns
810  -------
811  gain : `float`
812  Relative weight to give the new solution when updating the model.
813  A value of 1.0 gives equal weight to both solutions.
814 
815  Raises
816  ------
817  ValueError
818  If ``len(convergenceList) != len(gainList)+1``.
819  """
820  nIter = len(convergenceList)
821  if nIter != len(gainList) + 1:
822  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
823  % (len(convergenceList), len(gainList)))
824 
825  if self.config.baseGain is None:
826  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
827  # The more subfilters being modeled, the lower the gain should be.
828  baseGain = 1./(self.config.dcrNumSubfilters - 1)
829  else:
830  baseGain = self.config.baseGain
831 
832  if self.config.useProgressiveGain and nIter > 2:
833  # To calculate the best gain to use, compare the past gains that have been used
834  # with the resulting convergences to estimate the best gain to use.
835  # Algorithmically, this is a Kalman filter.
836  # If forward modeling proceeds perfectly, the convergence metric should
837  # asymptotically approach a final value.
838  # We can estimate that value from the measured changes in convergence
839  # weighted by the gains used in each previous iteration.
840  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
841  for i in range(nIter - 1)]
842  # The convergence metric is strictly positive, so if the estimated final convergence is
843  # less than zero, force it to zero.
844  estFinalConv = np.array(estFinalConv)
845  estFinalConv[estFinalConv < 0] = 0
846  # Because the estimate may slowly change over time, only use the most recent measurements.
847  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
848  lastGain = gainList[-1]
849  lastConv = convergenceList[-2]
850  newConv = convergenceList[-1]
851  # The predicted convergence is the value we would get if the new model calculated
852  # in the previous iteration was perfect. Recall that the updated model that is
853  # actually used is the gain-weighted average of the new and old model,
854  # so the convergence would be similarly weighted.
855  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
856  # If the measured and predicted convergence are very close, that indicates
857  # that our forward model is accurate and we can use a more aggressive gain
858  # If the measured convergence is significantly worse (or better!) than predicted,
859  # that indicates that the model is not converging as expected and
860  # we should use a more conservative gain.
861  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
862  newGain = 1 - abs(delta)
863  # Average the gains to prevent oscillating solutions.
864  newGain = (newGain + lastGain)/2.
865  gain = max(baseGain, newGain)
866  else:
867  gain = baseGain
868  gainList.append(gain)
869  return gain
870 
871  def calculateModelWeights(self, dcrModels, dcrBBox):
872  """Build an array that smoothly tapers to 0 away from detected sources.
873 
874  Parameters
875  ----------
876  dcrModels : `lsst.pipe.tasks.DcrModel`
877  Best fit model of the true sky after correcting chromatic effects.
878  dcrBBox : `lsst.afw.geom.box.Box2I`
879  Sub-region of the coadd which includes a buffer to allow for DCR.
880 
881  Returns
882  -------
883  weights : `numpy.ndarray` or `float`
884  A 2D array of weight values that tapers smoothly to zero away from detected sources.
885  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
886 
887  Raises
888  ------
889  ValueError
890  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
891  """
892  if self.config.modelWeightsWidth < 0:
893  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
894  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
895  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
896  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
897  weights[convergeMaskPixels] = 1.
898  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
899  weights /= np.max(weights)
900  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:159
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)