lsst.pipe.tasks  16.0-64-gd8935e55
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
34 
35 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
36 
37 
39  dcrNumSubfilters = pexConfig.Field(
40  dtype=int,
41  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
42  default=3,
43  )
44  maxNumIter = pexConfig.Field(
45  dtype=int,
46  doc="Maximum number of iterations of forward modeling.",
47  default=8,
48  )
49  minNumIter = pexConfig.Field(
50  dtype=int,
51  doc="Minimum number of iterations of forward modeling.",
52  default=3,
53  )
54  convergenceThreshold = pexConfig.Field(
55  dtype=float,
56  doc="Target relative change in convergence between iterations of forward modeling.",
57  default=0.001,
58  )
59  useConvergence = pexConfig.Field(
60  dtype=bool,
61  doc="Use convergence test as a forward modeling end condition?"
62  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
63  default=True,
64  )
65  baseGain = pexConfig.Field(
66  dtype=float,
67  optional=True,
68  doc="Relative weight to give the new solution vs. the last solution when updating the model."
69  "A value of 1.0 gives equal weight to both solutions."
70  "Small values imply slower convergence of the solution, but can "
71  "help prevent overshooting and failures in the fit."
72  "If ``baseGain`` is None, a conservative gain "
73  "will be calculated from the number of subfilters. ",
74  default=None,
75  )
76  useProgressiveGain = pexConfig.Field(
77  dtype=bool,
78  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
79  "When calculating the next gain, we use up to 5 previous gains and convergence values."
80  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
81  default=True,
82  )
83  doAirmassWeight = pexConfig.Field(
84  dtype=bool,
85  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
86  default=True,
87  )
88  modelWeightsWidth = pexConfig.Field(
89  dtype=float,
90  doc="Width of the region around detected sources to include in the DcrModel.",
91  default=3,
92  )
93  useModelWeights = pexConfig.Field(
94  dtype=bool,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=True,
97  )
98  splitSubfilters = pexConfig.Field(
99  dtype=bool,
100  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
101  "Instead of at the midpoint",
102  default=False,
103  )
104  regularizeModelIterations = pexConfig.Field(
105  dtype=float,
106  doc="Maximum relative change of the model allowed between iterations."
107  "Set to zero to disable.",
108  default=2.,
109  )
110  regularizeModelFrequency = pexConfig.Field(
111  dtype=float,
112  doc="Maximum relative change of the model allowed between subfilters."
113  "Set to zero to disable.",
114  default=4.,
115  )
116  convergenceMaskPlanes = pexConfig.ListField(
117  dtype=str,
118  default=["DETECTED"],
119  doc="Mask planes to use to calculate convergence."
120  )
121  regularizationWidth = pexConfig.Field(
122  dtype=int,
123  default=2,
124  doc="Minimum radius of a region to include in regularization, in pixels."
125  )
126  imageWarpMethod = pexConfig.Field(
127  dtype=str,
128  doc="Name of the warping kernel to use for shifting the image and variance planes.",
129  default="lanczos3",
130  )
131  maskWarpMethod = pexConfig.Field(
132  dtype=str,
133  doc="Name of the warping kernel to use for shifting the mask plane.",
134  default="bilinear",
135  )
136 
137  def setDefaults(self):
138  CompareWarpAssembleCoaddConfig.setDefaults(self)
139  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
140  self.doNImage = True
141  self.warpType = 'direct'
142  self.assembleStaticSkyModel.warpType = self.warpType
143  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
144  self.assembleStaticSkyModel.doNImage = False
145  self.assembleStaticSkyModel.doWrite = False
146  self.statistic = 'MEAN'
147 
148 
150  """Assemble DCR coadded images from a set of warps.
151 
152  Attributes
153  ----------
154  bufferSize : `int`
155  The number of pixels to grow each subregion by to allow for DCR.
156  warpCtrl : `lsst.afw.math.WarpingControl`
157  Configuration settings for warping an image
158 
159  Notes
160  -----
161  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
162  Warps (also called coadded temporary exposures), including the effects of
163  Differential Chromatic Refraction (DCR).
164  For full details of the mathematics and algorithm, please see
165  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
166 
167  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
168  each subfilter used in the iterative calculation.
169  It begins by dividing the bandpass-defining filter into N equal bandwidth
170  "subfilters", and divides the flux in each pixel from an initial coadd
171  equally into each as a "dcrModel". Because the airmass and parallactic
172  angle of each individual exposure is known, we can calculate the shift
173  relative to the center of the band in each subfilter due to DCR. For each
174  exposure we apply this shift as a linear transformation to the dcrModels
175  and stack the results to produce a DCR-matched exposure. The matched
176  exposures are subtracted from the input exposures to produce a set of
177  residual images, and these residuals are reverse shifted for each
178  exposures' subfilters and stacked. The shifted and stacked residuals are
179  added to the dcrModels to produce a new estimate of the flux in each pixel
180  within each subfilter. The dcrModels are solved for iteratively, which
181  continues until the solution from a new iteration improves by less than
182  a set percentage, or a maximum number of iterations is reached.
183  Two forms of regularization are employed to reduce unphysical results.
184  First, the new solution is averaged with the solution from the previous
185  iteration, which mitigates oscillating solutions where the model
186  overshoots with alternating very high and low values.
187  Second, a common degeneracy when the data have a limited range of airmass or
188  parallactic angle values is for one subfilter to be fit with very low or
189  negative values, while another subfilter is fit with very high values. This
190  typically appears in the form of holes next to sources in one subfilter,
191  and corresponding extended wings in another. Because each subfilter has
192  a narrow bandwidth we assume that physical sources that are above the noise
193  level will not vary in flux by more than a factor of `frequencyClampFactor`
194  between subfilters, and pixels that have flux deviations larger than that
195  factor will have the excess flux distributed evenly among all subfilters.
196  """
197 
198  ConfigClass = DcrAssembleCoaddConfig
199  _DefaultName = "dcrAssembleCoadd"
200 
201  @pipeBase.timeMethod
202  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
203  """Assemble a coadd from a set of warps.
204 
205  Coadd a set of Warps. Compute weights to be applied to each Warp and
206  find scalings to match the photometric zeropoint to a reference Warp.
207  Assemble the Warps using run method.
208  Forward model chromatic effects across multiple subfilters,
209  and subtract from the input Warps to build sets of residuals.
210  Use the residuals to construct a new ``DcrModel`` for each subfilter,
211  and iterate until the model converges.
212  Interpolate over NaNs and optionally write the coadd to disk.
213  Return the coadded exposure.
214 
215  Parameters
216  ----------
217  dataRef : `lsst.daf.persistence.ButlerDataRef`
218  Data reference defining the patch for coaddition and the
219  reference Warp
220  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
221  List of data references to warps. Data to be coadded will be
222  selected from this list based on overlap with the patch defined by
223  the data reference.
224 
225  Returns
226  -------
227  results : `lsst.pipe.base.Struct`
228  The Struct contains the following fields:
229 
230  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
231  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
232  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
233  - ``dcrNImages``: `list` of exposure count images for each subfilter
234  """
235  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
236  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
237 
238  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
239  warpRefList=warpRefList)
240  for subfilter in range(self.config.dcrNumSubfilters):
241  self.processResults(results.dcrCoadds[subfilter], dataRef)
242  if self.config.doWrite:
243  self.log.info("Persisting dcrCoadd")
244  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
245  numSubfilters=self.config.dcrNumSubfilters)
246  if self.config.doNImage and results.dcrNImages is not None:
247  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
248  numSubfilters=self.config.dcrNumSubfilters)
249 
250  return results
251 
252  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
253  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
254 
255  Sets the properties ``warpCtrl`` and ``bufferSize``.
256 
257  Parameters
258  ----------
259  templateCoadd : `lsst.afw.image.ExposureF`
260  The initial coadd exposure before accounting for DCR.
261  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
262  The data references to the input warped exposures.
263  weightList : `list` of `float`
264  The weight to give each input exposure in the coadd
265  Will be modified in place if ``doAirmassWeight`` is set.
266 
267  Returns
268  -------
269  dcrModels : `lsst.pipe.tasks.DcrModel`
270  Best fit model of the true sky after correcting chromatic effects.
271 
272  Raises
273  ------
274  NotImplementedError
275  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
276  """
277  filterInfo = templateCoadd.getFilter()
278  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
279  raise NotImplementedError("No minimum/maximum wavelength information found"
280  " in the filter definition! Please add lambdaMin and lambdaMax"
281  " to the Mapper class in your obs package.")
282  tempExpName = self.getTempExpDatasetName(self.warpType)
283  dcrShifts = []
284  for visitNum, tempExpRef in enumerate(tempExpRefList):
285  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
286  airmass = visitInfo.getBoresightAirmass()
287  if self.config.doAirmassWeight:
288  weightList[visitNum] *= airmass
289  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
290  filterInfo, self.config.dcrNumSubfilters))))
291  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
292  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
293  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
294  warpCache = 0
295  warpInterpLength = max(self.config.subregionSize)
296  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
297  self.config.maskWarpMethod,
298  cacheSize=warpCache, interpLength=warpInterpLength)
299  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
300  self.config.dcrNumSubfilters,
301  filterInfo=filterInfo,
302  psf=templateCoadd.getPsf())
303  return dcrModels
304 
305  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
306  supplementaryData=None):
307  """Assemble the coadd.
308 
309  Requires additional inputs Struct ``supplementaryData`` to contain a
310  ``templateCoadd`` that serves as the model of the static sky.
311 
312  Find artifacts and apply them to the warps' masks creating a list of
313  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
314  Then pass these alternative masks to the base class's assemble method.
315 
316  Divide the ``templateCoadd`` evenly between each subfilter of a
317  ``DcrModel`` as the starting best estimate of the true wavelength-
318  dependent sky. Forward model the ``DcrModel`` using the known
319  chromatic effects in each subfilter and calculate a convergence metric
320  based on how well the modeled template matches the input warps. If
321  the convergence has not yet reached the desired threshold, then shift
322  and stack the residual images to build a new ``DcrModel``. Apply
323  conditioning to prevent oscillating solutions between iterations or
324  between subfilters.
325 
326  Once the ``DcrModel`` reaches convergence or the maximum number of
327  iterations has been reached, fill the metadata for each subfilter
328  image and make them proper ``coaddExposure``s.
329 
330  Parameters
331  ----------
332  skyInfo : `lsst.pipe.base.Struct`
333  Patch geometry information, from getSkyInfo
334  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
335  The data references to the input warped exposures.
336  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
337  The image scalars correct for the zero point of the exposures.
338  weightList : `list` of `float`
339  The weight to give each input exposure in the coadd
340  supplementaryData : `lsst.pipe.base.Struct`
341  Result struct returned by ``makeSupplementaryData`` with components:
342 
343  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
344 
345  Returns
346  -------
347  result : `lsst.pipe.base.Struct`
348  Result struct with components:
349 
350  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
351  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
352  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
353  - ``dcrNImages``: `list` of exposure count images for each subfilter
354  """
355  templateCoadd = supplementaryData.templateCoadd
356  baseMask = templateCoadd.mask.clone()
357  # The variance plane is for each subfilter
358  # and should be proportionately lower than the full-band image
359  baseVariance = templateCoadd.variance.clone()
360  baseVariance /= self.config.dcrNumSubfilters
361  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
362  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
363  templateCoadd.setMask(baseMask)
364  badMaskPlanes = self.config.badMaskPlanes[:]
365  badMaskPlanes.append("CLIPPED")
366  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
367 
368  stats = self.prepareStats(mask=badPixelMask)
369  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
370  if self.config.doNImage:
371  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
372  tempExpRefList, spanSetMaskList, stats.ctrl)
373  nImage = afwImage.ImageU(skyInfo.bbox)
374  # Note that this nImage will be a factor of dcrNumSubfilters higher than
375  # the nImage returned by assembleCoadd for most pixels. This is because each
376  # subfilter may have a different nImage, and fractional values are not allowed.
377  for dcrNImage in dcrNImages:
378  nImage += dcrNImage
379  else:
380  dcrNImages = None
381 
382  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
383  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
384  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
385  subIter = 0
386  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
387  modelIter = 0
388  subIter += 1
389  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
390  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
391  dcrBBox = afwGeom.Box2I(subBBox)
392  dcrBBox.grow(self.bufferSize)
393  dcrBBox.clip(dcrModels.bbox)
394  if self.config.useModelWeights:
395  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
396  else:
397  modelWeights = 1.
398  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
399  imageScalerList, weightList, spanSetMaskList,
400  stats.ctrl)
401  self.log.info("Initial convergence : %s", convergenceMetric)
402  convergenceList = [convergenceMetric]
403  gainList = []
404  convergenceCheck = 1.
405  subfilterVariance = None
406  while (convergenceCheck > self.config.convergenceThreshold or
407  modelIter < self.config.minNumIter):
408  gain = self.calculateGain(convergenceList, gainList)
409  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
410  weightList, spanSetMaskList, stats.flags, stats.ctrl,
411  convergenceMetric, baseMask, subfilterVariance, gain,
412  modelWeights)
413  if self.config.useConvergence:
414  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
415  imageScalerList, weightList,
416  spanSetMaskList,
417  stats.ctrl)
418  if convergenceMetric == 0:
419  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
420  "most likely due to there being no valid data in the region.",
421  skyInfo.patchInfo.getIndex(), subIter)
422  break
423  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
424  if convergenceCheck < 0:
425  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
426  "iterations or desired convergence improvement of %s."
427  " Divergence: %s",
428  skyInfo.patchInfo.getIndex(), subIter,
429  self.config.convergenceThreshold, convergenceCheck)
430  break
431  convergenceList.append(convergenceMetric)
432  if modelIter > self.config.maxNumIter:
433  if self.config.useConvergence:
434  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
435  "before reaching desired convergence improvement of %s."
436  " Final convergence improvement: %s",
437  skyInfo.patchInfo.getIndex(), subIter,
438  self.config.convergenceThreshold, convergenceCheck)
439  break
440 
441  if self.config.useConvergence:
442  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
443  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
444  modelIter += 1
445  else:
446  if self.config.useConvergence:
447  self.log.info("Coadd patch %s subregion %s finished with "
448  "convergence metric %s after %s iterations",
449  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
450  else:
451  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
452  skyInfo.patchInfo.getIndex(), subIter, modelIter)
453  if self.config.useConvergence and convergenceMetric > 0:
454  self.log.info("Final convergence improvement was %.4f%% overall",
455  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
456 
457  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
458  calibration=self.scaleZeroPoint.getCalib(),
459  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
460  mask=baseMask,
461  variance=baseVariance)
462  coaddExposure = self.stackCoadd(dcrCoadds)
463  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
464  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
465 
466  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
467  """Calculate the number of exposures contributing to each subfilter.
468 
469  Parameters
470  ----------
471  dcrModels : `lsst.pipe.tasks.DcrModel`
472  Best fit model of the true sky after correcting chromatic effects.
473  bbox : `lsst.afw.geom.box.Box2I`
474  Bounding box of the patch to coadd.
475  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
476  The data references to the input warped exposures.
477  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
478  Each element is dict with keys = mask plane name to add the spans to
479  statsCtrl : `lsst.afw.math.StatisticsControl`
480  Statistics control object for coadd
481 
482  Returns
483  -------
484  dcrNImages : `list` of `lsst.afw.image.ImageU`
485  List of exposure count images for each subfilter
486  """
487  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
488  tempExpName = self.getTempExpDatasetName(self.warpType)
489  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
490  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
491  visitInfo = exposure.getInfo().getVisitInfo()
492  wcs = exposure.getInfo().getWcs()
493  mask = exposure.mask
494  if altMaskSpans is not None:
495  self.applyAltMaskPlanes(mask, altMaskSpans)
496  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
497  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
498  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
499  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
500  return dcrNImages
501 
502  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
503  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
504  baseMask, subfilterVariance, gain, modelWeights):
505  """Assemble the DCR coadd for a sub-region.
506 
507  Build a DCR-matched template for each input exposure, then shift the
508  residuals according to the DCR in each subfilter.
509  Stack the shifted residuals and apply them as a correction to the
510  solution from the previous iteration.
511  Restrict the new model solutions from varying by more than a factor of
512  `modelClampFactor` from the last solution, and additionally restrict the
513  individual subfilter models from varying by more than a factor of
514  `frequencyClampFactor` from their average.
515  Finally, mitigate potentially oscillating solutions by averaging the new
516  solution with the solution from the previous iteration, weighted by
517  their convergence metric.
518 
519  Parameters
520  ----------
521  dcrModels : `lsst.pipe.tasks.DcrModel`
522  Best fit model of the true sky after correcting chromatic effects.
523  bbox : `lsst.afw.geom.box.Box2I`
524  Bounding box of the subregion to coadd.
525  dcrBBox :`lsst.afw.geom.box.Box2I`
526  Sub-region of the coadd which includes a buffer to allow for DCR.
527  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
528  The data references to the input warped exposures.
529  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
530  The image scalars correct for the zero point of the exposures.
531  weightList : `list` of `float`
532  The weight to give each input exposure in the coadd
533  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
534  Each element is dict with keys = mask plane name to add the spans to
535  statsFlags : `lsst.afw.math.Property`
536  Statistics settings for coaddition.
537  statsCtrl : `lsst.afw.math.StatisticsControl`
538  Statistics control object for coadd
539  convergenceMetric : `float`
540  Quality of fit metric for the matched templates of the input images.
541  baseMask : `lsst.afw.image.Mask`
542  Mask of the initial template coadd.
543  subfilterVariance : `list` of `numpy.ndarray`
544  The variance of each coadded subfilter image.
545  gain : `float`, optional
546  Relative weight to give the new solution when updating the model.
547  modelWeights : `numpy.ndarray` or `float`
548  A 2D array of weight values that tapers smoothly to zero away from detected sources.
549  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
550  """
551  tempExpName = self.getTempExpDatasetName(self.warpType)
552  residualGeneratorList = []
553 
554  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
555  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
556  visitInfo = exposure.getInfo().getVisitInfo()
557  wcs = exposure.getInfo().getWcs()
558  maskedImage = exposure.maskedImage
559  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
560  bbox=dcrBBox, wcs=wcs, mask=baseMask,
561  splitSubfilters=self.config.splitSubfilters)
562  imageScaler.scaleMaskedImage(maskedImage)
563  if altMaskSpans is not None:
564  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
565 
566  if self.config.removeMaskPlanes:
567  self.removeMaskPlanes(maskedImage)
568  maskedImage -= templateImage
569  maskedImage.image.array *= modelWeights
570  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
571  dcrModels.filter))
572 
573  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
574  statsFlags, statsCtrl, weightList,
575  mask=baseMask, gain=gain)
576  dcrModels.assign(dcrSubModelOut, bbox)
577 
578  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
579  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
580 
581  Parameters
582  ----------
583  residual : `lsst.afw.image.MaskedImageF`
584  The residual masked image for one exposure,
585  after subtracting the matched template
586  visitInfo : `lsst.afw.image.VisitInfo`
587  Metadata for the exposure.
588  bbox : `lsst.afw.geom.box.Box2I`
589  Sub-region of the coadd
590  wcs : `lsst.afw.geom.SkyWcs`
591  Coordinate system definition (wcs) for the exposure.
592  filterInfo : `lsst.afw.image.Filter`
593  The filter definition, set in the current instruments' obs package.
594  Required for any calculation of DCR, including making matched templates.
595 
596  Yields
597  ------
598  residualImage : `lsst.afw.image.maskedImageF`
599  The residual image for the next subfilter, shifted for DCR.
600  """
601  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
602  for dcr in dcrShift:
603  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
604 
605  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
606  statsFlags, statsCtrl, weightList,
607  mask, gain):
608  """Calculate a new DcrModel from a set of image residuals.
609 
610  Parameters
611  ----------
612  dcrModels : `lsst.pipe.tasks.DcrModel`
613  Current model of the true sky after correcting chromatic effects.
614  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
615  The residual image for the next subfilter, shifted for DCR.
616  bbox : `lsst.afw.geom.box.Box2I`
617  Sub-region of the coadd
618  statsFlags : `lsst.afw.math.Property`
619  Statistics settings for coaddition.
620  statsCtrl : `lsst.afw.math.StatisticsControl`
621  Statistics control object for coadd
622  weightList : `list` of `float`
623  The weight to give each input exposure in the coadd
624  mask : `lsst.afw.image.Mask`
625  Mask to use for each new model image.
626  gain : `float`
627  Relative weight to give the new solution when updating the model.
628 
629  Returns
630  -------
631  dcrModel : `lsst.pipe.tasks.DcrModel`
632  New model of the true sky after correcting chromatic effects.
633  """
634  maskMap = self.setRejectedMaskMapping(statsCtrl)
635  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
636  newModelImages = []
637  for subfilter, model in enumerate(dcrModels):
638  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
639  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
640  clipped, maskMap)
641  residual.setXY0(bbox.getBegin())
642  # `MaskedImage`s only support in-place addition, so rename for readability
643  residual += model[bbox]
644  newModel = residual
645  # Catch any invalid values
646  badPixels = ~np.isfinite(newModel.image.array)
647  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
648  # every iteration, masked regions will continually expand.
649  newModel.setMask(mask[bbox])
650  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
651  if self.config.regularizeModelIterations > 0:
652  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
653  self.config.regularizeModelIterations,
654  self.config.regularizationWidth)
655  newModelImages.append(newModel)
656  if self.config.regularizeModelFrequency > 0:
657  dcrModels.regularizeModelFreq(newModelImages, bbox,
658  self.config.regularizeModelFrequency,
659  self.config.regularizationWidth)
660  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
661  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
662 
663  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
664  weightList, spanSetMaskList, statsCtrl):
665  """Calculate a quality of fit metric for the matched templates.
666 
667  Parameters
668  ----------
669  dcrModels : `lsst.pipe.tasks.DcrModel`
670  Best fit model of the true sky after correcting chromatic effects.
671  bbox : `lsst.afw.geom.box.Box2I`
672  Sub-region to coadd
673  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
674  The data references to the input warped exposures.
675  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
676  The image scalars correct for the zero point of the exposures.
677  weightList : `list` of `float`
678  The weight to give each input exposure in the coadd
679  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
680  Each element is dict with keys = mask plane name to add the spans to
681  statsCtrl : `lsst.afw.math.StatisticsControl`
682  Statistics control object for coadd
683 
684  Returns
685  -------
686  convergenceMetric : `float`
687  Quality of fit metric for all input exposures, within the sub-region
688  """
689  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
690  nSigma = 3.
691  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
692  bufferSize=self.bufferSize)
693  tempExpName = self.getTempExpDatasetName(self.warpType)
694  weight = 0
695  metric = 0.
696  metricList = {}
697  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
698  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
699  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
700  imageScaler.scaleMaskedImage(exposure.maskedImage)
701  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
702  altMaskSpans=altMaskSpans)
703  metric += singleMetric*expWeight
704  metricList[tempExpRef.dataId["visit"]] = singleMetric
705  weight += expWeight
706  self.log.info("Individual metrics:\n%s", metricList)
707  return 1.0 if weight == 0.0 else metric/weight
708 
709  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
710  statsCtrl, altMaskSpans=None):
711  """Calculate a quality of fit metric for a single matched template.
712 
713  Parameters
714  ----------
715  dcrModels : `lsst.pipe.tasks.DcrModel`
716  Best fit model of the true sky after correcting chromatic effects.
717  exposure : `lsst.afw.image.ExposureF`
718  The input warped exposure to evaluate.
719  significanceImage : `numpy.ndarray`
720  Array of weights for each pixel corresponding to its significance
721  for the convergence calculation.
722  statsCtrl : `lsst.afw.math.StatisticsControl`
723  Statistics control object for coadd
724  altMaskSpans : `dict` containing spanSet lists, or None
725  The keys of the `dict` equal the mask plane name to add the spans to
726 
727  Returns
728  -------
729  convergenceMetric : `float`
730  Quality of fit metric for one exposure, within the sub-region.
731  """
732  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
733  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
734  visitInfo=exposure.getInfo().getVisitInfo(),
735  bbox=exposure.getBBox(),
736  wcs=exposure.getInfo().getWcs())
737  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
738  refVals = np.abs(templateImage.image.array)*significanceImage
739 
740  finitePixels = np.isfinite(diffVals)
741  if altMaskSpans is not None:
742  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
743  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
744  convergeMaskPixels = exposure.mask.array & convergeMask > 0
745  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
746  if np.sum(usePixels) == 0:
747  metric = 0.
748  else:
749  diffUse = diffVals[usePixels]
750  refUse = refVals[usePixels]
751  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
752  return metric
753 
754  def stackCoadd(self, dcrCoadds):
755  """Add a list of sub-band coadds together.
756 
757  Parameters
758  ----------
759  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
760  A list of coadd exposures, each exposure containing
761  the model for one subfilter.
762 
763  Returns
764  -------
765  coaddExposure : `lsst.afw.image.ExposureF`
766  A single coadd exposure that is the sum of the sub-bands.
767  """
768  coaddExposure = dcrCoadds[0].clone()
769  for coadd in dcrCoadds[1:]:
770  coaddExposure.maskedImage += coadd.maskedImage
771  return coaddExposure
772 
773  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
774  mask=None, variance=None):
775  """Create a list of coadd exposures from a list of masked images.
776 
777  Parameters
778  ----------
779  dcrModels : `lsst.pipe.tasks.DcrModel`
780  Best fit model of the true sky after correcting chromatic effects.
781  skyInfo : `lsst.pipe.base.Struct`
782  Patch geometry information, from getSkyInfo
783  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
784  The data references to the input warped exposures.
785  weightList : `list` of `float`
786  The weight to give each input exposure in the coadd
787  calibration : `lsst.afw.Image.Calib`, optional
788  Scale factor to set the photometric zero point of an exposure.
789  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
790  A record of the observations that are included in the coadd.
791  mask : `lsst.afw.image.Mask`, optional
792  Optional mask to override the values in the final coadd.
793  variance : `lsst.afw.image.Image`, optional
794  Optional variance plane to override the values in the final coadd.
795 
796  Returns
797  -------
798  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
799  A list of coadd exposures, each exposure containing
800  the model for one subfilter.
801  """
802  dcrCoadds = []
803  for model in dcrModels:
804  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
805  if calibration is not None:
806  coaddExposure.setCalib(calibration)
807  if coaddInputs is not None:
808  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
809  # Set the metadata for the coadd, including PSF and aperture corrections.
810  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
811  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
812  coaddExposure.setMaskedImage(model[skyInfo.bbox])
813  if mask is not None:
814  coaddExposure.setMask(mask)
815  if variance is not None:
816  coaddExposure.setVariance(variance)
817  dcrCoadds.append(coaddExposure)
818  return dcrCoadds
819 
820  def calculateGain(self, convergenceList, gainList):
821  """Calculate the gain to use for the current iteration.
822 
823  After calculating a new DcrModel, each value is averaged with the
824  value in the corresponding pixel from the previous iteration. This
825  reduces oscillating solutions that iterative techniques are plagued by,
826  and speeds convergence. By far the biggest changes to the model
827  happen in the first couple iterations, so we can also use a more
828  aggressive gain later when the model is changing slowly.
829 
830  Parameters
831  ----------
832  convergenceList : `list` of `float`
833  The quality of fit metric from each previous iteration.
834  gainList : `list` of `float`
835  The gains used in each previous iteration: appended with the new
836  gain value.
837  Gains are numbers between ``self.config.baseGain`` and 1.
838 
839  Returns
840  -------
841  gain : `float`
842  Relative weight to give the new solution when updating the model.
843  A value of 1.0 gives equal weight to both solutions.
844 
845  Raises
846  ------
847  ValueError
848  If ``len(convergenceList) != len(gainList)+1``.
849  """
850  nIter = len(convergenceList)
851  if nIter != len(gainList) + 1:
852  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
853  % (len(convergenceList), len(gainList)))
854 
855  if self.config.baseGain is None:
856  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
857  # The more subfilters being modeled, the lower the gain should be.
858  baseGain = 1./(self.config.dcrNumSubfilters - 1)
859  else:
860  baseGain = self.config.baseGain
861 
862  if self.config.useProgressiveGain and nIter > 2:
863  # To calculate the best gain to use, compare the past gains that have been used
864  # with the resulting convergences to estimate the best gain to use.
865  # Algorithmically, this is a Kalman filter.
866  # If forward modeling proceeds perfectly, the convergence metric should
867  # asymptotically approach a final value.
868  # We can estimate that value from the measured changes in convergence
869  # weighted by the gains used in each previous iteration.
870  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
871  for i in range(nIter - 1)]
872  # The convergence metric is strictly positive, so if the estimated final convergence is
873  # less than zero, force it to zero.
874  estFinalConv = np.array(estFinalConv)
875  estFinalConv[estFinalConv < 0] = 0
876  # Because the estimate may slowly change over time, only use the most recent measurements.
877  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
878  lastGain = gainList[-1]
879  lastConv = convergenceList[-2]
880  newConv = convergenceList[-1]
881  # The predicted convergence is the value we would get if the new model calculated
882  # in the previous iteration was perfect. Recall that the updated model that is
883  # actually used is the gain-weighted average of the new and old model,
884  # so the convergence would be similarly weighted.
885  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
886  # If the measured and predicted convergence are very close, that indicates
887  # that our forward model is accurate and we can use a more aggressive gain
888  # If the measured convergence is significantly worse (or better!) than predicted,
889  # that indicates that the model is not converging as expected and
890  # we should use a more conservative gain.
891  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
892  newGain = 1 - abs(delta)
893  # Average the gains to prevent oscillating solutions.
894  newGain = (newGain + lastGain)/2.
895  gain = max(baseGain, newGain)
896  else:
897  gain = baseGain
898  gainList.append(gain)
899  return gain
900 
901  def calculateModelWeights(self, dcrModels, dcrBBox):
902  """Build an array that smoothly tapers to 0 away from detected sources.
903 
904  Parameters
905  ----------
906  dcrModels : `lsst.pipe.tasks.DcrModel`
907  Best fit model of the true sky after correcting chromatic effects.
908  dcrBBox : `lsst.afw.geom.box.Box2I`
909  Sub-region of the coadd which includes a buffer to allow for DCR.
910 
911  Returns
912  -------
913  weights : `numpy.ndarray` or `float`
914  A 2D array of weight values that tapers smoothly to zero away from detected sources.
915  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
916 
917  Raises
918  ------
919  ValueError
920  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
921  """
922  if self.config.modelWeightsWidth < 0:
923  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
924  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
925  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
926  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
927  weights[convergeMaskPixels] = 1.
928  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
929  weights /= np.max(weights)
930  return weights
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)