lsst.pipe.tasks  16.0-37-gdefe6db9
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from scipy import ndimage
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  doc="Maximum number of iterations of forward modeling.",
46  default=8,
47  )
48  minNumIter = pexConfig.Field(
49  dtype=int,
50  doc="Minimum number of iterations of forward modeling.",
51  default=3,
52  )
53  convergenceThreshold = pexConfig.Field(
54  dtype=float,
55  doc="Target relative change in convergence between iterations of forward modeling.",
56  default=0.001,
57  )
58  useConvergence = pexConfig.Field(
59  dtype=bool,
60  doc="Use convergence test as a forward modeling end condition?"
61  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
62  default=True,
63  )
64  baseGain = pexConfig.Field(
65  dtype=float,
66  optional=True,
67  doc="Relative weight to give the new solution vs. the last solution when updating the model."
68  "A value of 1.0 gives equal weight to both solutions."
69  "Small values imply slower convergence of the solution, but can "
70  "help prevent overshooting and failures in the fit."
71  "If ``baseGain`` is None, a conservative gain "
72  "will be calculated from the number of subfilters. ",
73  default=None,
74  )
75  useProgressiveGain = pexConfig.Field(
76  dtype=bool,
77  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
78  "When calculating the next gain, we use up to 5 previous gains and convergence values."
79  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
80  default=True,
81  )
82  doAirmassWeight = pexConfig.Field(
83  dtype=bool,
84  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
85  default=True,
86  )
87  modelWeightsWidth = pexConfig.Field(
88  dtype=float,
89  doc="Width of the region around detected sources to include in the DcrModel.",
90  default=3,
91  )
92  useModelWeights = pexConfig.Field(
93  dtype=bool,
94  doc="Width of the region around detected sources to include in the DcrModel.",
95  default=True,
96  )
97  splitSubfilters = pexConfig.Field(
98  dtype=bool,
99  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
100  "Instead of at the midpoint",
101  default=False,
102  )
103  regularizeModelIterations = pexConfig.Field(
104  dtype=float,
105  doc="Maximum relative change of the model allowed between iterations."
106  "Set to zero to disable.",
107  default=2.,
108  )
109  regularizeModelFrequency = pexConfig.Field(
110  dtype=float,
111  doc="Maximum relative change of the model allowed between subfilters."
112  "Set to zero to disable.",
113  default=4.,
114  )
115  convergenceMaskPlanes = pexConfig.ListField(
116  dtype=str,
117  default=["DETECTED"],
118  doc="Mask planes to use to calculate convergence."
119  )
120  regularizationWidth = pexConfig.Field(
121  dtype=int,
122  default=2,
123  doc="Minimum radius of a region to include in regularization, in pixels."
124  )
125  imageWarpMethod = pexConfig.Field(
126  dtype=str,
127  doc="Name of the warping kernel to use for shifting the image and variance planes.",
128  default="lanczos3",
129  )
130  maskWarpMethod = pexConfig.Field(
131  dtype=str,
132  doc="Name of the warping kernel to use for shifting the mask plane.",
133  default="bilinear",
134  )
135 
136  def setDefaults(self):
137  CompareWarpAssembleCoaddConfig.setDefaults(self)
138  self.doNImage = True
139  self.warpType = 'direct'
140  self.assembleStaticSkyModel.warpType = self.warpType
141  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
142  self.assembleStaticSkyModel.doNImage = False
143  self.assembleStaticSkyModel.doWrite = False
144  self.statistic = 'MEAN'
145 
146 
148  """Assemble DCR coadded images from a set of warps.
149 
150  Attributes
151  ----------
152  bufferSize : `int`
153  The number of pixels to grow each subregion by to allow for DCR.
154  warpCtrl : `lsst.afw.math.WarpingControl`
155  Configuration settings for warping an image
156 
157  Notes
158  -----
159  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
160  Warps (also called coadded temporary exposures), including the effects of
161  Differential Chromatic Refraction (DCR).
162  For full details of the mathematics and algorithm, please see
163  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
164 
165  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
166  each subfilter used in the iterative calculation.
167  It begins by dividing the bandpass-defining filter into N equal bandwidth
168  "subfilters", and divides the flux in each pixel from an initial coadd
169  equally into each as a "dcrModel". Because the airmass and parallactic
170  angle of each individual exposure is known, we can calculate the shift
171  relative to the center of the band in each subfilter due to DCR. For each
172  exposure we apply this shift as a linear transformation to the dcrModels
173  and stack the results to produce a DCR-matched exposure. The matched
174  exposures are subtracted from the input exposures to produce a set of
175  residual images, and these residuals are reverse shifted for each
176  exposures' subfilters and stacked. The shifted and stacked residuals are
177  added to the dcrModels to produce a new estimate of the flux in each pixel
178  within each subfilter. The dcrModels are solved for iteratively, which
179  continues until the solution from a new iteration improves by less than
180  a set percentage, or a maximum number of iterations is reached.
181  Two forms of regularization are employed to reduce unphysical results.
182  First, the new solution is averaged with the solution from the previous
183  iteration, which mitigates oscillating solutions where the model
184  overshoots with alternating very high and low values.
185  Second, a common degeneracy when the data have a limited range of airmass or
186  parallactic angle values is for one subfilter to be fit with very low or
187  negative values, while another subfilter is fit with very high values. This
188  typically appears in the form of holes next to sources in one subfilter,
189  and corresponding extended wings in another. Because each subfilter has
190  a narrow bandwidth we assume that physical sources that are above the noise
191  level will not vary in flux by more than a factor of `frequencyClampFactor`
192  between subfilters, and pixels that have flux deviations larger than that
193  factor will have the excess flux distributed evenly among all subfilters.
194  """
195 
196  ConfigClass = DcrAssembleCoaddConfig
197  _DefaultName = "dcrAssembleCoadd"
198 
199  @pipeBase.timeMethod
200  def runDataRef(self, dataRef, selectDataList=[]):
201  """Assemble a coadd from a set of warps.
202 
203  Coadd a set of Warps. Compute weights to be applied to each Warp and
204  find scalings to match the photometric zeropoint to a reference Warp.
205  Assemble the Warps using run method.
206  Forward model chromatic effects across multiple subfilters,
207  and subtract from the input Warps to build sets of residuals.
208  Use the residuals to construct a new ``DcrModel`` for each subfilter,
209  and iterate until the model converges.
210  Interpolate over NaNs and optionally write the coadd to disk.
211  Return the coadded exposure.
212 
213  Parameters
214  ----------
215  dataRef : `lsst.daf.persistence.ButlerDataRef`
216  Data reference defining the patch for coaddition and the
217  reference Warp
218  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
219  List of data references to warps. Data to be coadded will be
220  selected from this list based on overlap with the patch defined by
221  the data reference.
222 
223  Returns
224  -------
225  results : `lsst.pipe.base.Struct`
226  The Struct contains the following fields:
227 
228  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
229  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
230  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
231  - ``dcrNImages``: `list` of exposure count images for each subfilter
232  """
233  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
234  for subfilter in range(self.config.dcrNumSubfilters):
235  self.processResults(results.dcrCoadds[subfilter], dataRef)
236  if self.config.doWrite:
237  self.log.info("Persisting dcrCoadd")
238  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
239  numSubfilters=self.config.dcrNumSubfilters)
240  if self.config.doNImage and results.dcrNImages is not None:
241  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
242  numSubfilters=self.config.dcrNumSubfilters)
243 
244  return results
245 
246  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
247  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
248 
249  Sets the properties ``warpCtrl`` and ``bufferSize``.
250 
251  Parameters
252  ----------
253  templateCoadd : `lsst.afw.image.ExposureF`
254  The initial coadd exposure before accounting for DCR.
255  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
256  The data references to the input warped exposures.
257  weightList : `list` of `float`
258  The weight to give each input exposure in the coadd
259  Will be modified in place if ``doAirmassWeight`` is set.
260 
261  Returns
262  -------
263  dcrModels : `lsst.pipe.tasks.DcrModel`
264  Best fit model of the true sky after correcting chromatic effects.
265 
266  Raises
267  ------
268  NotImplementedError
269  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
270  """
271  filterInfo = templateCoadd.getFilter()
272  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
273  raise NotImplementedError("No minimum/maximum wavelength information found"
274  " in the filter definition! Please add lambdaMin and lambdaMax"
275  " to the Mapper class in your obs package.")
276  tempExpName = self.getTempExpDatasetName(self.warpType)
277  dcrShifts = []
278  for visitNum, tempExpRef in enumerate(tempExpRefList):
279  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
280  airmass = visitInfo.getBoresightAirmass()
281  if self.config.doAirmassWeight:
282  weightList[visitNum] *= airmass
283  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
284  filterInfo, self.config.dcrNumSubfilters))))
285  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
286  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
287  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
288  warpCache = 0
289  warpInterpLength = max(self.config.subregionSize)
290  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
291  self.config.maskWarpMethod,
292  cacheSize=warpCache, interpLength=warpInterpLength)
293  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
294  self.config.dcrNumSubfilters,
295  filterInfo=filterInfo,
296  psf=templateCoadd.getPsf())
297  return dcrModels
298 
299  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
300  supplementaryData=None):
301  """Assemble the coadd.
302 
303  Requires additional inputs Struct ``supplementaryData`` to contain a
304  ``templateCoadd`` that serves as the model of the static sky.
305 
306  Find artifacts and apply them to the warps' masks creating a list of
307  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
308  Then pass these alternative masks to the base class's assemble method.
309 
310  Divide the ``templateCoadd`` evenly between each subfilter of a
311  ``DcrModel`` as the starting best estimate of the true wavelength-
312  dependent sky. Forward model the ``DcrModel`` using the known
313  chromatic effects in each subfilter and calculate a convergence metric
314  based on how well the modeled template matches the input warps. If
315  the convergence has not yet reached the desired threshold, then shift
316  and stack the residual images to build a new ``DcrModel``. Apply
317  conditioning to prevent oscillating solutions between iterations or
318  between subfilters.
319 
320  Once the ``DcrModel`` reaches convergence or the maximum number of
321  iterations has been reached, fill the metadata for each subfilter
322  image and make them proper ``coaddExposure``s.
323 
324  Parameters
325  ----------
326  skyInfo : `lsst.pipe.base.Struct`
327  Patch geometry information, from getSkyInfo
328  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
329  The data references to the input warped exposures.
330  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
331  The image scalars correct for the zero point of the exposures.
332  weightList : `list` of `float`
333  The weight to give each input exposure in the coadd
334  supplementaryData : `lsst.pipe.base.Struct`
335  Result struct returned by ``makeSupplementaryData`` with components:
336 
337  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
338 
339  Returns
340  -------
341  result : `lsst.pipe.base.Struct`
342  Result struct with components:
343 
344  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
345  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
346  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
347  - ``dcrNImages``: `list` of exposure count images for each subfilter
348  """
349  templateCoadd = supplementaryData.templateCoadd
350  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
351  badMaskPlanes = self.config.badMaskPlanes[:]
352  badMaskPlanes.append("CLIPPED")
353  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
354  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
355  # Psf-Matching moves the real edge inwards
356  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
357 
358  stats = self.prepareStats(mask=badPixelMask)
359  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
360  if self.config.doNImage:
361  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
362  tempExpRefList, spanSetMaskList, stats.ctrl)
363  nImage = afwImage.ImageU(skyInfo.bbox)
364  # Note that this nImage will be a factor of dcrNumSubfilters higher than
365  # the nImage returned by assembleCoadd for most pixels. This is because each
366  # subfilter may have a different nImage, and fractional values are not allowed.
367  for dcrNImage in dcrNImages:
368  nImage += dcrNImage
369  else:
370  dcrNImages = None
371 
372  baseMask = templateCoadd.mask
373  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
374  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
375  modelIter = 0
376  self.log.info("Computing coadd over %s", subBBox)
377  dcrBBox = afwGeom.Box2I(subBBox)
378  dcrBBox.grow(self.bufferSize)
379  dcrBBox.clip(dcrModels.bbox)
380  if self.config.useModelWeights:
381  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
382  else:
383  modelWeights = 1.
384  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
385  imageScalerList, weightList, spanSetMaskList,
386  stats.ctrl)
387  self.log.info("Initial convergence : %s", convergenceMetric)
388  convergenceList = [convergenceMetric]
389  gainList = []
390  convergenceCheck = 1.
391  subfilterVariance = None
392  while (convergenceCheck > self.config.convergenceThreshold or
393  modelIter < self.config.minNumIter):
394  gain = self.calculateGain(convergenceList, gainList)
395  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
396  weightList, spanSetMaskList, stats.flags, stats.ctrl,
397  convergenceMetric, baseMask, subfilterVariance, gain,
398  modelWeights)
399  if self.config.useConvergence:
400  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
401  imageScalerList, weightList,
402  spanSetMaskList,
403  stats.ctrl)
404  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
405  if convergenceCheck < 0:
406  self.log.warn("Coadd %s diverged before reaching maximum iterations or"
407  " desired convergence improvement of %s."
408  " Divergence: %s",
409  subBBox, self.config.convergenceThreshold, convergenceCheck)
410  break
411  convergenceList.append(convergenceMetric)
412  if modelIter > self.config.maxNumIter:
413  if self.config.useConvergence:
414  self.log.warn("Coadd %s reached maximum iterations before reaching"
415  " desired convergence improvement of %s."
416  " Final convergence improvement: %s",
417  subBBox, self.config.convergenceThreshold, convergenceCheck)
418  break
419 
420  if self.config.useConvergence:
421  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
422  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
423  modelIter += 1
424  else:
425  if self.config.useConvergence:
426  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
427  subBBox, convergenceMetric, modelIter)
428  else:
429  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
430  if self.config.useConvergence:
431  self.log.info("Final convergence improvement was %.4f%% overall",
432  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
433 
434  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
435  calibration=self.scaleZeroPoint.getCalib(),
436  coaddInputs=self.inputRecorder.makeCoaddInputs(),
437  mask=templateCoadd.mask)
438  coaddExposure = self.stackCoadd(dcrCoadds)
439  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
440  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
441 
442  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
443  """Calculate the number of exposures contributing to each subfilter.
444 
445  Parameters
446  ----------
447  dcrModels : `lsst.pipe.tasks.DcrModel`
448  Best fit model of the true sky after correcting chromatic effects.
449  bbox : `lsst.afw.geom.box.Box2I`
450  Bounding box of the patch to coadd.
451  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
452  The data references to the input warped exposures.
453  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
454  Each element is dict with keys = mask plane name to add the spans to
455  statsCtrl : `lsst.afw.math.StatisticsControl`
456  Statistics control object for coadd
457 
458  Returns
459  -------
460  dcrNImages : `list` of `lsst.afw.image.ImageU`
461  List of exposure count images for each subfilter
462  """
463  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
464  tempExpName = self.getTempExpDatasetName(self.warpType)
465  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
466  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
467  visitInfo = exposure.getInfo().getVisitInfo()
468  wcs = exposure.getInfo().getWcs()
469  mask = exposure.mask
470  if altMaskSpans is not None:
471  self.applyAltMaskPlanes(mask, altMaskSpans)
472  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
473  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
474  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
475  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
476  return dcrNImages
477 
478  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
479  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
480  baseMask, subfilterVariance, gain, modelWeights):
481  """Assemble the DCR coadd for a sub-region.
482 
483  Build a DCR-matched template for each input exposure, then shift the
484  residuals according to the DCR in each subfilter.
485  Stack the shifted residuals and apply them as a correction to the
486  solution from the previous iteration.
487  Restrict the new model solutions from varying by more than a factor of
488  `modelClampFactor` from the last solution, and additionally restrict the
489  individual subfilter models from varying by more than a factor of
490  `frequencyClampFactor` from their average.
491  Finally, mitigate potentially oscillating solutions by averaging the new
492  solution with the solution from the previous iteration, weighted by
493  their convergence metric.
494 
495  Parameters
496  ----------
497  dcrModels : `lsst.pipe.tasks.DcrModel`
498  Best fit model of the true sky after correcting chromatic effects.
499  bbox : `lsst.afw.geom.box.Box2I`
500  Bounding box of the subregion to coadd.
501  dcrBBox :`lsst.afw.geom.box.Box2I`
502  Sub-region of the coadd which includes a buffer to allow for DCR.
503  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
504  The data references to the input warped exposures.
505  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
506  The image scalars correct for the zero point of the exposures.
507  weightList : `list` of `float`
508  The weight to give each input exposure in the coadd
509  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
510  Each element is dict with keys = mask plane name to add the spans to
511  statsFlags : `lsst.afw.math.Property`
512  Statistics settings for coaddition.
513  statsCtrl : `lsst.afw.math.StatisticsControl`
514  Statistics control object for coadd
515  convergenceMetric : `float`
516  Quality of fit metric for the matched templates of the input images.
517  baseMask : `lsst.afw.image.Mask`
518  Mask of the initial template coadd.
519  subfilterVariance : `list` of `numpy.ndarray`
520  The variance of each coadded subfilter image.
521  gain : `float`, optional
522  Relative weight to give the new solution when updating the model.
523  modelWeights : `numpy.ndarray` or `float`
524  A 2D array of weight values that tapers smoothly to zero away from detected sources.
525  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
526  """
527  tempExpName = self.getTempExpDatasetName(self.warpType)
528  residualGeneratorList = []
529 
530  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
531  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
532  visitInfo = exposure.getInfo().getVisitInfo()
533  wcs = exposure.getInfo().getWcs()
534  maskedImage = exposure.maskedImage
535  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
536  bbox=dcrBBox, wcs=wcs, mask=baseMask,
537  splitSubfilters=self.config.splitSubfilters)
538  imageScaler.scaleMaskedImage(maskedImage)
539  if altMaskSpans is not None:
540  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
541 
542  if self.config.removeMaskPlanes:
543  self.removeMaskPlanes(maskedImage)
544  maskedImage -= templateImage
545  maskedImage.image.array *= modelWeights
546  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
547  dcrModels.filter))
548 
549  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
550  statsFlags, statsCtrl, weightList,
551  mask=baseMask, gain=gain)
552  dcrModels.assign(dcrSubModelOut, bbox)
553 
554  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
555  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
556 
557  Parameters
558  ----------
559  residual : `lsst.afw.image.MaskedImageF`
560  The residual masked image for one exposure,
561  after subtracting the matched template
562  visitInfo : `lsst.afw.image.VisitInfo`
563  Metadata for the exposure.
564  bbox : `lsst.afw.geom.box.Box2I`
565  Sub-region of the coadd
566  wcs : `lsst.afw.geom.SkyWcs`
567  Coordinate system definition (wcs) for the exposure.
568  filterInfo : `lsst.afw.image.Filter`
569  The filter definition, set in the current instruments' obs package.
570  Required for any calculation of DCR, including making matched templates.
571 
572  Yields
573  ------
574  residualImage : `lsst.afw.image.maskedImageF`
575  The residual image for the next subfilter, shifted for DCR.
576  """
577  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
578  for dcr in dcrShift:
579  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
580 
581  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
582  statsFlags, statsCtrl, weightList,
583  mask, gain):
584  """Calculate a new DcrModel from a set of image residuals.
585 
586  Parameters
587  ----------
588  dcrModels : `lsst.pipe.tasks.DcrModel`
589  Current model of the true sky after correcting chromatic effects.
590  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
591  The residual image for the next subfilter, shifted for DCR.
592  bbox : `lsst.afw.geom.box.Box2I`
593  Sub-region of the coadd
594  statsFlags : `lsst.afw.math.Property`
595  Statistics settings for coaddition.
596  statsCtrl : `lsst.afw.math.StatisticsControl`
597  Statistics control object for coadd
598  weightList : `list` of `float`
599  The weight to give each input exposure in the coadd
600  mask : `lsst.afw.image.Mask`
601  Mask to use for each new model image.
602  gain : `float`
603  Relative weight to give the new solution when updating the model.
604 
605  Returns
606  -------
607  dcrModel : `lsst.pipe.tasks.DcrModel`
608  New model of the true sky after correcting chromatic effects.
609  """
610  maskMap = self.setRejectedMaskMapping(statsCtrl)
611  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
612  newModelImages = []
613  for subfilter, model in enumerate(dcrModels):
614  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
615  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
616  clipped, maskMap)
617  residual.setXY0(bbox.getBegin())
618  # `MaskedImage`s only support in-place addition, so rename for readability
619  residual += model[bbox]
620  newModel = residual
621  # Catch any invalid values
622  badPixels = ~np.isfinite(newModel.image.array)
623  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
624  # every iteration, masked regions will continually expand.
625  newModel.setMask(mask[bbox])
626  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
627  if self.config.regularizeModelIterations > 0:
628  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
629  self.config.regularizeModelIterations,
630  self.config.regularizationWidth)
631  newModelImages.append(newModel)
632  if self.config.regularizeModelFrequency > 0:
633  dcrModels.regularizeModelFreq(newModelImages, bbox,
634  self.config.regularizeModelFrequency,
635  self.config.regularizationWidth)
636  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
637  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
638 
639  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
640  weightList, spanSetMaskList, statsCtrl):
641  """Calculate a quality of fit metric for the matched templates.
642 
643  Parameters
644  ----------
645  dcrModels : `lsst.pipe.tasks.DcrModel`
646  Best fit model of the true sky after correcting chromatic effects.
647  bbox : `lsst.afw.geom.box.Box2I`
648  Sub-region to coadd
649  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
650  The data references to the input warped exposures.
651  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
652  The image scalars correct for the zero point of the exposures.
653  weightList : `list` of `float`
654  The weight to give each input exposure in the coadd
655  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
656  Each element is dict with keys = mask plane name to add the spans to
657  statsCtrl : `lsst.afw.math.StatisticsControl`
658  Statistics control object for coadd
659 
660  Returns
661  -------
662  convergenceMetric : `float`
663  Quality of fit metric for all input exposures, within the sub-region
664  """
665  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
666  nSigma = 3.
667  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
668  bufferSize=self.bufferSize)
669  tempExpName = self.getTempExpDatasetName(self.warpType)
670  weight = 0
671  metric = 0.
672  metricList = {}
673  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
674  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
675  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
676  imageScaler.scaleMaskedImage(exposure.maskedImage)
677  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
678  altMaskSpans=altMaskSpans)
679  metric += singleMetric*expWeight
680  metricList[tempExpRef.dataId["visit"]] = singleMetric
681  weight += expWeight
682  self.log.info("Individual metrics:\n%s", metricList)
683  return 1.0 if weight == 0.0 else metric/weight
684 
685  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
686  statsCtrl, altMaskSpans=None):
687  """Calculate a quality of fit metric for a single matched template.
688 
689  Parameters
690  ----------
691  dcrModels : `lsst.pipe.tasks.DcrModel`
692  Best fit model of the true sky after correcting chromatic effects.
693  exposure : `lsst.afw.image.ExposureF`
694  The input warped exposure to evaluate.
695  significanceImage : `numpy.ndarray`
696  Array of weights for each pixel corresponding to its significance
697  for the convergence calculation.
698  statsCtrl : `lsst.afw.math.StatisticsControl`
699  Statistics control object for coadd
700  altMaskSpans : `dict` containing spanSet lists, or None
701  The keys of the `dict` equal the mask plane name to add the spans to
702 
703  Returns
704  -------
705  convergenceMetric : `float`
706  Quality of fit metric for one exposure, within the sub-region.
707  """
708  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
709  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
710  visitInfo=exposure.getInfo().getVisitInfo(),
711  bbox=exposure.getBBox(),
712  wcs=exposure.getInfo().getWcs())
713  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
714  refVals = np.abs(templateImage.image.array)*significanceImage
715 
716  finitePixels = np.isfinite(diffVals)
717  if altMaskSpans is not None:
718  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
719  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
720  convergeMaskPixels = exposure.mask.array & convergeMask > 0
721  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
722  if np.sum(usePixels) == 0:
723  metric = 0.
724  else:
725  diffUse = diffVals[usePixels]
726  refUse = refVals[usePixels]
727  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
728  return metric
729 
730  def stackCoadd(self, dcrCoadds):
731  """Add a list of sub-band coadds together.
732 
733  Parameters
734  ----------
735  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
736  A list of coadd exposures, each exposure containing
737  the model for one subfilter.
738 
739  Returns
740  -------
741  coaddExposure : `lsst.afw.image.ExposureF`
742  A single coadd exposure that is the sum of the sub-bands.
743  """
744  coaddExposure = dcrCoadds[0].clone()
745  for coadd in dcrCoadds[1:]:
746  coaddExposure.maskedImage += coadd.maskedImage
747  return coaddExposure
748 
749  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
750  mask=None):
751  """Create a list of coadd exposures from a list of masked images.
752 
753  Parameters
754  ----------
755  dcrModels : `lsst.pipe.tasks.DcrModel`
756  Best fit model of the true sky after correcting chromatic effects.
757  skyInfo : `lsst.pipe.base.Struct`
758  Patch geometry information, from getSkyInfo
759  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
760  The data references to the input warped exposures.
761  weightList : `list` of `float`
762  The weight to give each input exposure in the coadd
763  calibration : `lsst.afw.Image.Calib`, optional
764  Scale factor to set the photometric zero point of an exposure.
765  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
766  A record of the observations that are included in the coadd.
767  mask : `lsst.afw.image.Mask`, optional
768  Optional mask to override the values in the final coadd.
769 
770  Returns
771  -------
772  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
773  A list of coadd exposures, each exposure containing
774  the model for one subfilter.
775  """
776  dcrCoadds = []
777  for model in dcrModels:
778  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
779  if calibration is not None:
780  coaddExposure.setCalib(calibration)
781  if coaddInputs is not None:
782  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
783  # Set the metadata for the coadd, including PSF and aperture corrections.
784  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
785  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
786  coaddExposure.setMaskedImage(model[skyInfo.bbox])
787  if mask is not None:
788  coaddExposure.setMask(mask)
789  dcrCoadds.append(coaddExposure)
790  return dcrCoadds
791 
792  def calculateGain(self, convergenceList, gainList):
793  """Calculate the gain to use for the current iteration.
794 
795  After calculating a new DcrModel, each value is averaged with the
796  value in the corresponding pixel from the previous iteration. This
797  reduces oscillating solutions that iterative techniques are plagued by,
798  and speeds convergence. By far the biggest changes to the model
799  happen in the first couple iterations, so we can also use a more
800  aggressive gain later when the model is changing slowly.
801 
802  Parameters
803  ----------
804  convergenceList : `list` of `float`
805  The quality of fit metric from each previous iteration.
806  gainList : `list` of `float`
807  The gains used in each previous iteration: appended with the new
808  gain value.
809  Gains are numbers between ``self.config.baseGain`` and 1.
810 
811  Returns
812  -------
813  gain : `float`
814  Relative weight to give the new solution when updating the model.
815  A value of 1.0 gives equal weight to both solutions.
816 
817  Raises
818  ------
819  ValueError
820  If ``len(convergenceList) != len(gainList)+1``.
821  """
822  nIter = len(convergenceList)
823  if nIter != len(gainList) + 1:
824  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
825  % (len(convergenceList), len(gainList)))
826 
827  if self.config.baseGain is None:
828  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
829  # The more subfilters being modeled, the lower the gain should be.
830  baseGain = 1./(self.config.dcrNumSubfilters - 1)
831  else:
832  baseGain = self.config.baseGain
833 
834  if self.config.useProgressiveGain and nIter > 2:
835  # To calculate the best gain to use, compare the past gains that have been used
836  # with the resulting convergences to estimate the best gain to use.
837  # Algorithmically, this is a Kalman filter.
838  # If forward modeling proceeds perfectly, the convergence metric should
839  # asymptotically approach a final value.
840  # We can estimate that value from the measured changes in convergence
841  # weighted by the gains used in each previous iteration.
842  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
843  for i in range(nIter - 1)]
844  # The convergence metric is strictly positive, so if the estimated final convergence is
845  # less than zero, force it to zero.
846  estFinalConv = np.array(estFinalConv)
847  estFinalConv[estFinalConv < 0] = 0
848  # Because the estimate may slowly change over time, only use the most recent measurements.
849  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
850  lastGain = gainList[-1]
851  lastConv = convergenceList[-2]
852  newConv = convergenceList[-1]
853  # The predicted convergence is the value we would get if the new model calculated
854  # in the previous iteration was perfect. Recall that the updated model that is
855  # actually used is the gain-weighted average of the new and old model,
856  # so the convergence would be similarly weighted.
857  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
858  # If the measured and predicted convergence are very close, that indicates
859  # that our forward model is accurate and we can use a more aggressive gain
860  # If the measured convergence is significantly worse (or better!) than predicted,
861  # that indicates that the model is not converging as expected and
862  # we should use a more conservative gain.
863  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
864  newGain = 1 - abs(delta)
865  # Average the gains to prevent oscillating solutions.
866  newGain = (newGain + lastGain)/2.
867  gain = max(baseGain, newGain)
868  else:
869  gain = baseGain
870  gainList.append(gain)
871  return gain
872 
873  def calculateModelWeights(self, dcrModels, dcrBBox):
874  """Build an array that smoothly tapers to 0 away from detected sources.
875 
876  Parameters
877  ----------
878  dcrModels : `lsst.pipe.tasks.DcrModel`
879  Best fit model of the true sky after correcting chromatic effects.
880  dcrBBox : `lsst.afw.geom.box.Box2I`
881  Sub-region of the coadd which includes a buffer to allow for DCR.
882 
883  Returns
884  -------
885  weights : `numpy.ndarray` or `float`
886  A 2D array of weight values that tapers smoothly to zero away from detected sources.
887  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
888 
889  Raises
890  ------
891  ValueError
892  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
893  """
894  if self.config.modelWeightsWidth < 0:
895  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
896  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
897  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
898  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
899  weights[convergeMaskPixels] = 1.
900  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
901  weights /= np.max(weights)
902  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:159
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)