lsst.pipe.tasks  16.0-49-g42e581f7+7
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
34 
35 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
36 
37 
39  dcrNumSubfilters = pexConfig.Field(
40  dtype=int,
41  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
42  default=3,
43  )
44  maxNumIter = pexConfig.Field(
45  dtype=int,
46  doc="Maximum number of iterations of forward modeling.",
47  default=8,
48  )
49  minNumIter = pexConfig.Field(
50  dtype=int,
51  doc="Minimum number of iterations of forward modeling.",
52  default=3,
53  )
54  convergenceThreshold = pexConfig.Field(
55  dtype=float,
56  doc="Target relative change in convergence between iterations of forward modeling.",
57  default=0.001,
58  )
59  useConvergence = pexConfig.Field(
60  dtype=bool,
61  doc="Use convergence test as a forward modeling end condition?"
62  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
63  default=True,
64  )
65  baseGain = pexConfig.Field(
66  dtype=float,
67  optional=True,
68  doc="Relative weight to give the new solution vs. the last solution when updating the model."
69  "A value of 1.0 gives equal weight to both solutions."
70  "Small values imply slower convergence of the solution, but can "
71  "help prevent overshooting and failures in the fit."
72  "If ``baseGain`` is None, a conservative gain "
73  "will be calculated from the number of subfilters. ",
74  default=None,
75  )
76  useProgressiveGain = pexConfig.Field(
77  dtype=bool,
78  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
79  "When calculating the next gain, we use up to 5 previous gains and convergence values."
80  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
81  default=True,
82  )
83  doAirmassWeight = pexConfig.Field(
84  dtype=bool,
85  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
86  default=True,
87  )
88  modelWeightsWidth = pexConfig.Field(
89  dtype=float,
90  doc="Width of the region around detected sources to include in the DcrModel.",
91  default=3,
92  )
93  useModelWeights = pexConfig.Field(
94  dtype=bool,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=True,
97  )
98  splitSubfilters = pexConfig.Field(
99  dtype=bool,
100  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
101  "Instead of at the midpoint",
102  default=False,
103  )
104  regularizeModelIterations = pexConfig.Field(
105  dtype=float,
106  doc="Maximum relative change of the model allowed between iterations."
107  "Set to zero to disable.",
108  default=2.,
109  )
110  regularizeModelFrequency = pexConfig.Field(
111  dtype=float,
112  doc="Maximum relative change of the model allowed between subfilters."
113  "Set to zero to disable.",
114  default=4.,
115  )
116  convergenceMaskPlanes = pexConfig.ListField(
117  dtype=str,
118  default=["DETECTED"],
119  doc="Mask planes to use to calculate convergence."
120  )
121  regularizationWidth = pexConfig.Field(
122  dtype=int,
123  default=2,
124  doc="Minimum radius of a region to include in regularization, in pixels."
125  )
126  imageWarpMethod = pexConfig.Field(
127  dtype=str,
128  doc="Name of the warping kernel to use for shifting the image and variance planes.",
129  default="lanczos3",
130  )
131  maskWarpMethod = pexConfig.Field(
132  dtype=str,
133  doc="Name of the warping kernel to use for shifting the mask plane.",
134  default="bilinear",
135  )
136 
137  def setDefaults(self):
138  CompareWarpAssembleCoaddConfig.setDefaults(self)
139  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
140  self.doNImage = True
141  self.warpType = 'direct'
142  self.assembleStaticSkyModel.warpType = self.warpType
143  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
144  self.assembleStaticSkyModel.doNImage = False
145  self.assembleStaticSkyModel.doWrite = False
146  self.statistic = 'MEAN'
147 
148 
150  """Assemble DCR coadded images from a set of warps.
151 
152  Attributes
153  ----------
154  bufferSize : `int`
155  The number of pixels to grow each subregion by to allow for DCR.
156  warpCtrl : `lsst.afw.math.WarpingControl`
157  Configuration settings for warping an image
158 
159  Notes
160  -----
161  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
162  Warps (also called coadded temporary exposures), including the effects of
163  Differential Chromatic Refraction (DCR).
164  For full details of the mathematics and algorithm, please see
165  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
166 
167  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
168  each subfilter used in the iterative calculation.
169  It begins by dividing the bandpass-defining filter into N equal bandwidth
170  "subfilters", and divides the flux in each pixel from an initial coadd
171  equally into each as a "dcrModel". Because the airmass and parallactic
172  angle of each individual exposure is known, we can calculate the shift
173  relative to the center of the band in each subfilter due to DCR. For each
174  exposure we apply this shift as a linear transformation to the dcrModels
175  and stack the results to produce a DCR-matched exposure. The matched
176  exposures are subtracted from the input exposures to produce a set of
177  residual images, and these residuals are reverse shifted for each
178  exposures' subfilters and stacked. The shifted and stacked residuals are
179  added to the dcrModels to produce a new estimate of the flux in each pixel
180  within each subfilter. The dcrModels are solved for iteratively, which
181  continues until the solution from a new iteration improves by less than
182  a set percentage, or a maximum number of iterations is reached.
183  Two forms of regularization are employed to reduce unphysical results.
184  First, the new solution is averaged with the solution from the previous
185  iteration, which mitigates oscillating solutions where the model
186  overshoots with alternating very high and low values.
187  Second, a common degeneracy when the data have a limited range of airmass or
188  parallactic angle values is for one subfilter to be fit with very low or
189  negative values, while another subfilter is fit with very high values. This
190  typically appears in the form of holes next to sources in one subfilter,
191  and corresponding extended wings in another. Because each subfilter has
192  a narrow bandwidth we assume that physical sources that are above the noise
193  level will not vary in flux by more than a factor of `frequencyClampFactor`
194  between subfilters, and pixels that have flux deviations larger than that
195  factor will have the excess flux distributed evenly among all subfilters.
196  """
197 
198  ConfigClass = DcrAssembleCoaddConfig
199  _DefaultName = "dcrAssembleCoadd"
200 
201  @pipeBase.timeMethod
202  def runDataRef(self, dataRef, selectDataList=[]):
203  """Assemble a coadd from a set of warps.
204 
205  Coadd a set of Warps. Compute weights to be applied to each Warp and
206  find scalings to match the photometric zeropoint to a reference Warp.
207  Assemble the Warps using run method.
208  Forward model chromatic effects across multiple subfilters,
209  and subtract from the input Warps to build sets of residuals.
210  Use the residuals to construct a new ``DcrModel`` for each subfilter,
211  and iterate until the model converges.
212  Interpolate over NaNs and optionally write the coadd to disk.
213  Return the coadded exposure.
214 
215  Parameters
216  ----------
217  dataRef : `lsst.daf.persistence.ButlerDataRef`
218  Data reference defining the patch for coaddition and the
219  reference Warp
220  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
221  List of data references to warps. Data to be coadded will be
222  selected from this list based on overlap with the patch defined by
223  the data reference.
224 
225  Returns
226  -------
227  results : `lsst.pipe.base.Struct`
228  The Struct contains the following fields:
229 
230  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
231  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
232  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
233  - ``dcrNImages``: `list` of exposure count images for each subfilter
234  """
235  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
236  for subfilter in range(self.config.dcrNumSubfilters):
237  self.processResults(results.dcrCoadds[subfilter], dataRef)
238  if self.config.doWrite:
239  self.log.info("Persisting dcrCoadd")
240  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
241  numSubfilters=self.config.dcrNumSubfilters)
242  if self.config.doNImage and results.dcrNImages is not None:
243  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
244  numSubfilters=self.config.dcrNumSubfilters)
245 
246  return results
247 
248  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
249  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
250 
251  Sets the properties ``warpCtrl`` and ``bufferSize``.
252 
253  Parameters
254  ----------
255  templateCoadd : `lsst.afw.image.ExposureF`
256  The initial coadd exposure before accounting for DCR.
257  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
258  The data references to the input warped exposures.
259  weightList : `list` of `float`
260  The weight to give each input exposure in the coadd
261  Will be modified in place if ``doAirmassWeight`` is set.
262 
263  Returns
264  -------
265  dcrModels : `lsst.pipe.tasks.DcrModel`
266  Best fit model of the true sky after correcting chromatic effects.
267 
268  Raises
269  ------
270  NotImplementedError
271  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
272  """
273  filterInfo = templateCoadd.getFilter()
274  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
275  raise NotImplementedError("No minimum/maximum wavelength information found"
276  " in the filter definition! Please add lambdaMin and lambdaMax"
277  " to the Mapper class in your obs package.")
278  tempExpName = self.getTempExpDatasetName(self.warpType)
279  dcrShifts = []
280  for visitNum, tempExpRef in enumerate(tempExpRefList):
281  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
282  airmass = visitInfo.getBoresightAirmass()
283  if self.config.doAirmassWeight:
284  weightList[visitNum] *= airmass
285  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
286  filterInfo, self.config.dcrNumSubfilters))))
287  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
288  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
289  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
290  warpCache = 0
291  warpInterpLength = max(self.config.subregionSize)
292  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
293  self.config.maskWarpMethod,
294  cacheSize=warpCache, interpLength=warpInterpLength)
295  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
296  self.config.dcrNumSubfilters,
297  filterInfo=filterInfo,
298  psf=templateCoadd.getPsf())
299  return dcrModels
300 
301  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
302  supplementaryData=None):
303  """Assemble the coadd.
304 
305  Requires additional inputs Struct ``supplementaryData`` to contain a
306  ``templateCoadd`` that serves as the model of the static sky.
307 
308  Find artifacts and apply them to the warps' masks creating a list of
309  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
310  Then pass these alternative masks to the base class's assemble method.
311 
312  Divide the ``templateCoadd`` evenly between each subfilter of a
313  ``DcrModel`` as the starting best estimate of the true wavelength-
314  dependent sky. Forward model the ``DcrModel`` using the known
315  chromatic effects in each subfilter and calculate a convergence metric
316  based on how well the modeled template matches the input warps. If
317  the convergence has not yet reached the desired threshold, then shift
318  and stack the residual images to build a new ``DcrModel``. Apply
319  conditioning to prevent oscillating solutions between iterations or
320  between subfilters.
321 
322  Once the ``DcrModel`` reaches convergence or the maximum number of
323  iterations has been reached, fill the metadata for each subfilter
324  image and make them proper ``coaddExposure``s.
325 
326  Parameters
327  ----------
328  skyInfo : `lsst.pipe.base.Struct`
329  Patch geometry information, from getSkyInfo
330  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
331  The data references to the input warped exposures.
332  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
333  The image scalars correct for the zero point of the exposures.
334  weightList : `list` of `float`
335  The weight to give each input exposure in the coadd
336  supplementaryData : `lsst.pipe.base.Struct`
337  Result struct returned by ``makeSupplementaryData`` with components:
338 
339  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
340 
341  Returns
342  -------
343  result : `lsst.pipe.base.Struct`
344  Result struct with components:
345 
346  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
347  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
348  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
349  - ``dcrNImages``: `list` of exposure count images for each subfilter
350  """
351  templateCoadd = supplementaryData.templateCoadd
352  baseMask = templateCoadd.mask.clone()
353  # The variance plane is for each subfilter
354  # and should be proportionately lower than the full-band image
355  baseVariance = templateCoadd.variance.clone()
356  baseVariance /= self.config.dcrNumSubfilters
357  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
358  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
359  templateCoadd.setMask(baseMask)
360  badMaskPlanes = self.config.badMaskPlanes[:]
361  badMaskPlanes.append("CLIPPED")
362  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
363 
364  stats = self.prepareStats(mask=badPixelMask)
365  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
366  if self.config.doNImage:
367  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
368  tempExpRefList, spanSetMaskList, stats.ctrl)
369  nImage = afwImage.ImageU(skyInfo.bbox)
370  # Note that this nImage will be a factor of dcrNumSubfilters higher than
371  # the nImage returned by assembleCoadd for most pixels. This is because each
372  # subfilter may have a different nImage, and fractional values are not allowed.
373  for dcrNImage in dcrNImages:
374  nImage += dcrNImage
375  else:
376  dcrNImages = None
377 
378  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
379  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
380  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
381  subIter = 0
382  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
383  modelIter = 0
384  subIter += 1
385  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
386  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
387  dcrBBox = afwGeom.Box2I(subBBox)
388  dcrBBox.grow(self.bufferSize)
389  dcrBBox.clip(dcrModels.bbox)
390  if self.config.useModelWeights:
391  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
392  else:
393  modelWeights = 1.
394  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
395  imageScalerList, weightList, spanSetMaskList,
396  stats.ctrl)
397  self.log.info("Initial convergence : %s", convergenceMetric)
398  convergenceList = [convergenceMetric]
399  gainList = []
400  convergenceCheck = 1.
401  subfilterVariance = None
402  while (convergenceCheck > self.config.convergenceThreshold or
403  modelIter < self.config.minNumIter):
404  gain = self.calculateGain(convergenceList, gainList)
405  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
406  weightList, spanSetMaskList, stats.flags, stats.ctrl,
407  convergenceMetric, baseMask, subfilterVariance, gain,
408  modelWeights)
409  if self.config.useConvergence:
410  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
411  imageScalerList, weightList,
412  spanSetMaskList,
413  stats.ctrl)
414  if convergenceMetric == 0:
415  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
416  "most likely due to there being no valid data in the region.",
417  skyInfo.patchInfo.getIndex(), subIter)
418  break
419  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
420  if convergenceCheck < 0:
421  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
422  "iterations or desired convergence improvement of %s."
423  " Divergence: %s",
424  skyInfo.patchInfo.getIndex(), subIter,
425  self.config.convergenceThreshold, convergenceCheck)
426  break
427  convergenceList.append(convergenceMetric)
428  if modelIter > self.config.maxNumIter:
429  if self.config.useConvergence:
430  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
431  "before reaching desired convergence improvement of %s."
432  " Final convergence improvement: %s",
433  skyInfo.patchInfo.getIndex(), subIter,
434  self.config.convergenceThreshold, convergenceCheck)
435  break
436 
437  if self.config.useConvergence:
438  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
439  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
440  modelIter += 1
441  else:
442  if self.config.useConvergence:
443  self.log.info("Coadd patch %s subregion %s finished with "
444  "convergence metric %s after %s iterations",
445  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
446  else:
447  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
448  skyInfo.patchInfo.getIndex(), subIter, modelIter)
449  if self.config.useConvergence and convergenceMetric > 0:
450  self.log.info("Final convergence improvement was %.4f%% overall",
451  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
452 
453  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
454  calibration=self.scaleZeroPoint.getCalib(),
455  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
456  mask=baseMask,
457  variance=baseVariance)
458  coaddExposure = self.stackCoadd(dcrCoadds)
459  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
460  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
461 
462  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
463  """Calculate the number of exposures contributing to each subfilter.
464 
465  Parameters
466  ----------
467  dcrModels : `lsst.pipe.tasks.DcrModel`
468  Best fit model of the true sky after correcting chromatic effects.
469  bbox : `lsst.afw.geom.box.Box2I`
470  Bounding box of the patch to coadd.
471  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
472  The data references to the input warped exposures.
473  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
474  Each element is dict with keys = mask plane name to add the spans to
475  statsCtrl : `lsst.afw.math.StatisticsControl`
476  Statistics control object for coadd
477 
478  Returns
479  -------
480  dcrNImages : `list` of `lsst.afw.image.ImageU`
481  List of exposure count images for each subfilter
482  """
483  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
484  tempExpName = self.getTempExpDatasetName(self.warpType)
485  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
486  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
487  visitInfo = exposure.getInfo().getVisitInfo()
488  wcs = exposure.getInfo().getWcs()
489  mask = exposure.mask
490  if altMaskSpans is not None:
491  self.applyAltMaskPlanes(mask, altMaskSpans)
492  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
493  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
494  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
495  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
496  return dcrNImages
497 
498  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
499  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
500  baseMask, subfilterVariance, gain, modelWeights):
501  """Assemble the DCR coadd for a sub-region.
502 
503  Build a DCR-matched template for each input exposure, then shift the
504  residuals according to the DCR in each subfilter.
505  Stack the shifted residuals and apply them as a correction to the
506  solution from the previous iteration.
507  Restrict the new model solutions from varying by more than a factor of
508  `modelClampFactor` from the last solution, and additionally restrict the
509  individual subfilter models from varying by more than a factor of
510  `frequencyClampFactor` from their average.
511  Finally, mitigate potentially oscillating solutions by averaging the new
512  solution with the solution from the previous iteration, weighted by
513  their convergence metric.
514 
515  Parameters
516  ----------
517  dcrModels : `lsst.pipe.tasks.DcrModel`
518  Best fit model of the true sky after correcting chromatic effects.
519  bbox : `lsst.afw.geom.box.Box2I`
520  Bounding box of the subregion to coadd.
521  dcrBBox :`lsst.afw.geom.box.Box2I`
522  Sub-region of the coadd which includes a buffer to allow for DCR.
523  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
524  The data references to the input warped exposures.
525  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
526  The image scalars correct for the zero point of the exposures.
527  weightList : `list` of `float`
528  The weight to give each input exposure in the coadd
529  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
530  Each element is dict with keys = mask plane name to add the spans to
531  statsFlags : `lsst.afw.math.Property`
532  Statistics settings for coaddition.
533  statsCtrl : `lsst.afw.math.StatisticsControl`
534  Statistics control object for coadd
535  convergenceMetric : `float`
536  Quality of fit metric for the matched templates of the input images.
537  baseMask : `lsst.afw.image.Mask`
538  Mask of the initial template coadd.
539  subfilterVariance : `list` of `numpy.ndarray`
540  The variance of each coadded subfilter image.
541  gain : `float`, optional
542  Relative weight to give the new solution when updating the model.
543  modelWeights : `numpy.ndarray` or `float`
544  A 2D array of weight values that tapers smoothly to zero away from detected sources.
545  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
546  """
547  tempExpName = self.getTempExpDatasetName(self.warpType)
548  residualGeneratorList = []
549 
550  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
551  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
552  visitInfo = exposure.getInfo().getVisitInfo()
553  wcs = exposure.getInfo().getWcs()
554  maskedImage = exposure.maskedImage
555  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
556  bbox=dcrBBox, wcs=wcs, mask=baseMask,
557  splitSubfilters=self.config.splitSubfilters)
558  imageScaler.scaleMaskedImage(maskedImage)
559  if altMaskSpans is not None:
560  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
561 
562  if self.config.removeMaskPlanes:
563  self.removeMaskPlanes(maskedImage)
564  maskedImage -= templateImage
565  maskedImage.image.array *= modelWeights
566  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
567  dcrModels.filter))
568 
569  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
570  statsFlags, statsCtrl, weightList,
571  mask=baseMask, gain=gain)
572  dcrModels.assign(dcrSubModelOut, bbox)
573 
574  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
575  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
576 
577  Parameters
578  ----------
579  residual : `lsst.afw.image.MaskedImageF`
580  The residual masked image for one exposure,
581  after subtracting the matched template
582  visitInfo : `lsst.afw.image.VisitInfo`
583  Metadata for the exposure.
584  bbox : `lsst.afw.geom.box.Box2I`
585  Sub-region of the coadd
586  wcs : `lsst.afw.geom.SkyWcs`
587  Coordinate system definition (wcs) for the exposure.
588  filterInfo : `lsst.afw.image.Filter`
589  The filter definition, set in the current instruments' obs package.
590  Required for any calculation of DCR, including making matched templates.
591 
592  Yields
593  ------
594  residualImage : `lsst.afw.image.maskedImageF`
595  The residual image for the next subfilter, shifted for DCR.
596  """
597  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
598  for dcr in dcrShift:
599  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
600 
601  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
602  statsFlags, statsCtrl, weightList,
603  mask, gain):
604  """Calculate a new DcrModel from a set of image residuals.
605 
606  Parameters
607  ----------
608  dcrModels : `lsst.pipe.tasks.DcrModel`
609  Current model of the true sky after correcting chromatic effects.
610  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
611  The residual image for the next subfilter, shifted for DCR.
612  bbox : `lsst.afw.geom.box.Box2I`
613  Sub-region of the coadd
614  statsFlags : `lsst.afw.math.Property`
615  Statistics settings for coaddition.
616  statsCtrl : `lsst.afw.math.StatisticsControl`
617  Statistics control object for coadd
618  weightList : `list` of `float`
619  The weight to give each input exposure in the coadd
620  mask : `lsst.afw.image.Mask`
621  Mask to use for each new model image.
622  gain : `float`
623  Relative weight to give the new solution when updating the model.
624 
625  Returns
626  -------
627  dcrModel : `lsst.pipe.tasks.DcrModel`
628  New model of the true sky after correcting chromatic effects.
629  """
630  maskMap = self.setRejectedMaskMapping(statsCtrl)
631  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
632  newModelImages = []
633  for subfilter, model in enumerate(dcrModels):
634  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
635  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
636  clipped, maskMap)
637  residual.setXY0(bbox.getBegin())
638  # `MaskedImage`s only support in-place addition, so rename for readability
639  residual += model[bbox]
640  newModel = residual
641  # Catch any invalid values
642  badPixels = ~np.isfinite(newModel.image.array)
643  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
644  # every iteration, masked regions will continually expand.
645  newModel.setMask(mask[bbox])
646  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
647  if self.config.regularizeModelIterations > 0:
648  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
649  self.config.regularizeModelIterations,
650  self.config.regularizationWidth)
651  newModelImages.append(newModel)
652  if self.config.regularizeModelFrequency > 0:
653  dcrModels.regularizeModelFreq(newModelImages, bbox,
654  self.config.regularizeModelFrequency,
655  self.config.regularizationWidth)
656  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
657  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
658 
659  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
660  weightList, spanSetMaskList, statsCtrl):
661  """Calculate a quality of fit metric for the matched templates.
662 
663  Parameters
664  ----------
665  dcrModels : `lsst.pipe.tasks.DcrModel`
666  Best fit model of the true sky after correcting chromatic effects.
667  bbox : `lsst.afw.geom.box.Box2I`
668  Sub-region to coadd
669  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
670  The data references to the input warped exposures.
671  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
672  The image scalars correct for the zero point of the exposures.
673  weightList : `list` of `float`
674  The weight to give each input exposure in the coadd
675  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
676  Each element is dict with keys = mask plane name to add the spans to
677  statsCtrl : `lsst.afw.math.StatisticsControl`
678  Statistics control object for coadd
679 
680  Returns
681  -------
682  convergenceMetric : `float`
683  Quality of fit metric for all input exposures, within the sub-region
684  """
685  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
686  nSigma = 3.
687  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
688  bufferSize=self.bufferSize)
689  tempExpName = self.getTempExpDatasetName(self.warpType)
690  weight = 0
691  metric = 0.
692  metricList = {}
693  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
694  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
695  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
696  imageScaler.scaleMaskedImage(exposure.maskedImage)
697  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
698  altMaskSpans=altMaskSpans)
699  metric += singleMetric*expWeight
700  metricList[tempExpRef.dataId["visit"]] = singleMetric
701  weight += expWeight
702  self.log.info("Individual metrics:\n%s", metricList)
703  return 1.0 if weight == 0.0 else metric/weight
704 
705  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
706  statsCtrl, altMaskSpans=None):
707  """Calculate a quality of fit metric for a single matched template.
708 
709  Parameters
710  ----------
711  dcrModels : `lsst.pipe.tasks.DcrModel`
712  Best fit model of the true sky after correcting chromatic effects.
713  exposure : `lsst.afw.image.ExposureF`
714  The input warped exposure to evaluate.
715  significanceImage : `numpy.ndarray`
716  Array of weights for each pixel corresponding to its significance
717  for the convergence calculation.
718  statsCtrl : `lsst.afw.math.StatisticsControl`
719  Statistics control object for coadd
720  altMaskSpans : `dict` containing spanSet lists, or None
721  The keys of the `dict` equal the mask plane name to add the spans to
722 
723  Returns
724  -------
725  convergenceMetric : `float`
726  Quality of fit metric for one exposure, within the sub-region.
727  """
728  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
729  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
730  visitInfo=exposure.getInfo().getVisitInfo(),
731  bbox=exposure.getBBox(),
732  wcs=exposure.getInfo().getWcs())
733  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
734  refVals = np.abs(templateImage.image.array)*significanceImage
735 
736  finitePixels = np.isfinite(diffVals)
737  if altMaskSpans is not None:
738  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
739  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
740  convergeMaskPixels = exposure.mask.array & convergeMask > 0
741  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
742  if np.sum(usePixels) == 0:
743  metric = 0.
744  else:
745  diffUse = diffVals[usePixels]
746  refUse = refVals[usePixels]
747  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
748  return metric
749 
750  def stackCoadd(self, dcrCoadds):
751  """Add a list of sub-band coadds together.
752 
753  Parameters
754  ----------
755  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
756  A list of coadd exposures, each exposure containing
757  the model for one subfilter.
758 
759  Returns
760  -------
761  coaddExposure : `lsst.afw.image.ExposureF`
762  A single coadd exposure that is the sum of the sub-bands.
763  """
764  coaddExposure = dcrCoadds[0].clone()
765  for coadd in dcrCoadds[1:]:
766  coaddExposure.maskedImage += coadd.maskedImage
767  return coaddExposure
768 
769  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
770  mask=None, variance=None):
771  """Create a list of coadd exposures from a list of masked images.
772 
773  Parameters
774  ----------
775  dcrModels : `lsst.pipe.tasks.DcrModel`
776  Best fit model of the true sky after correcting chromatic effects.
777  skyInfo : `lsst.pipe.base.Struct`
778  Patch geometry information, from getSkyInfo
779  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
780  The data references to the input warped exposures.
781  weightList : `list` of `float`
782  The weight to give each input exposure in the coadd
783  calibration : `lsst.afw.Image.Calib`, optional
784  Scale factor to set the photometric zero point of an exposure.
785  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
786  A record of the observations that are included in the coadd.
787  mask : `lsst.afw.image.Mask`, optional
788  Optional mask to override the values in the final coadd.
789  variance : `lsst.afw.image.Image`, optional
790  Optional variance plane to override the values in the final coadd.
791 
792  Returns
793  -------
794  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
795  A list of coadd exposures, each exposure containing
796  the model for one subfilter.
797  """
798  dcrCoadds = []
799  for model in dcrModels:
800  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
801  if calibration is not None:
802  coaddExposure.setCalib(calibration)
803  if coaddInputs is not None:
804  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
805  # Set the metadata for the coadd, including PSF and aperture corrections.
806  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
807  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
808  coaddExposure.setMaskedImage(model[skyInfo.bbox])
809  if mask is not None:
810  coaddExposure.setMask(mask)
811  if variance is not None:
812  coaddExposure.setVariance(variance)
813  dcrCoadds.append(coaddExposure)
814  return dcrCoadds
815 
816  def calculateGain(self, convergenceList, gainList):
817  """Calculate the gain to use for the current iteration.
818 
819  After calculating a new DcrModel, each value is averaged with the
820  value in the corresponding pixel from the previous iteration. This
821  reduces oscillating solutions that iterative techniques are plagued by,
822  and speeds convergence. By far the biggest changes to the model
823  happen in the first couple iterations, so we can also use a more
824  aggressive gain later when the model is changing slowly.
825 
826  Parameters
827  ----------
828  convergenceList : `list` of `float`
829  The quality of fit metric from each previous iteration.
830  gainList : `list` of `float`
831  The gains used in each previous iteration: appended with the new
832  gain value.
833  Gains are numbers between ``self.config.baseGain`` and 1.
834 
835  Returns
836  -------
837  gain : `float`
838  Relative weight to give the new solution when updating the model.
839  A value of 1.0 gives equal weight to both solutions.
840 
841  Raises
842  ------
843  ValueError
844  If ``len(convergenceList) != len(gainList)+1``.
845  """
846  nIter = len(convergenceList)
847  if nIter != len(gainList) + 1:
848  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
849  % (len(convergenceList), len(gainList)))
850 
851  if self.config.baseGain is None:
852  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
853  # The more subfilters being modeled, the lower the gain should be.
854  baseGain = 1./(self.config.dcrNumSubfilters - 1)
855  else:
856  baseGain = self.config.baseGain
857 
858  if self.config.useProgressiveGain and nIter > 2:
859  # To calculate the best gain to use, compare the past gains that have been used
860  # with the resulting convergences to estimate the best gain to use.
861  # Algorithmically, this is a Kalman filter.
862  # If forward modeling proceeds perfectly, the convergence metric should
863  # asymptotically approach a final value.
864  # We can estimate that value from the measured changes in convergence
865  # weighted by the gains used in each previous iteration.
866  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
867  for i in range(nIter - 1)]
868  # The convergence metric is strictly positive, so if the estimated final convergence is
869  # less than zero, force it to zero.
870  estFinalConv = np.array(estFinalConv)
871  estFinalConv[estFinalConv < 0] = 0
872  # Because the estimate may slowly change over time, only use the most recent measurements.
873  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
874  lastGain = gainList[-1]
875  lastConv = convergenceList[-2]
876  newConv = convergenceList[-1]
877  # The predicted convergence is the value we would get if the new model calculated
878  # in the previous iteration was perfect. Recall that the updated model that is
879  # actually used is the gain-weighted average of the new and old model,
880  # so the convergence would be similarly weighted.
881  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
882  # If the measured and predicted convergence are very close, that indicates
883  # that our forward model is accurate and we can use a more aggressive gain
884  # If the measured convergence is significantly worse (or better!) than predicted,
885  # that indicates that the model is not converging as expected and
886  # we should use a more conservative gain.
887  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
888  newGain = 1 - abs(delta)
889  # Average the gains to prevent oscillating solutions.
890  newGain = (newGain + lastGain)/2.
891  gain = max(baseGain, newGain)
892  else:
893  gain = baseGain
894  gainList.append(gain)
895  return gain
896 
897  def calculateModelWeights(self, dcrModels, dcrBBox):
898  """Build an array that smoothly tapers to 0 away from detected sources.
899 
900  Parameters
901  ----------
902  dcrModels : `lsst.pipe.tasks.DcrModel`
903  Best fit model of the true sky after correcting chromatic effects.
904  dcrBBox : `lsst.afw.geom.box.Box2I`
905  Sub-region of the coadd which includes a buffer to allow for DCR.
906 
907  Returns
908  -------
909  weights : `numpy.ndarray` or `float`
910  A 2D array of weight values that tapers smoothly to zero away from detected sources.
911  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
912 
913  Raises
914  ------
915  ValueError
916  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
917  """
918  if self.config.modelWeightsWidth < 0:
919  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
920  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
921  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
922  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
923  weights[convergeMaskPixels] = 1.
924  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
925  weights /= np.max(weights)
926  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)