lsst.pipe.tasks  16.0-39-gcc1e6200
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.math as afwMath
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.pex.config as pexConfig
32 import lsst.pipe.base as pipeBase
33 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
34 
35 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
36 
37 
39  dcrNumSubfilters = pexConfig.Field(
40  dtype=int,
41  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
42  default=3,
43  )
44  maxNumIter = pexConfig.Field(
45  dtype=int,
46  doc="Maximum number of iterations of forward modeling.",
47  default=8,
48  )
49  minNumIter = pexConfig.Field(
50  dtype=int,
51  doc="Minimum number of iterations of forward modeling.",
52  default=3,
53  )
54  convergenceThreshold = pexConfig.Field(
55  dtype=float,
56  doc="Target relative change in convergence between iterations of forward modeling.",
57  default=0.001,
58  )
59  useConvergence = pexConfig.Field(
60  dtype=bool,
61  doc="Use convergence test as a forward modeling end condition?"
62  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
63  default=True,
64  )
65  baseGain = pexConfig.Field(
66  dtype=float,
67  optional=True,
68  doc="Relative weight to give the new solution vs. the last solution when updating the model."
69  "A value of 1.0 gives equal weight to both solutions."
70  "Small values imply slower convergence of the solution, but can "
71  "help prevent overshooting and failures in the fit."
72  "If ``baseGain`` is None, a conservative gain "
73  "will be calculated from the number of subfilters. ",
74  default=None,
75  )
76  useProgressiveGain = pexConfig.Field(
77  dtype=bool,
78  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
79  "When calculating the next gain, we use up to 5 previous gains and convergence values."
80  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
81  default=True,
82  )
83  doAirmassWeight = pexConfig.Field(
84  dtype=bool,
85  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
86  default=True,
87  )
88  modelWeightsWidth = pexConfig.Field(
89  dtype=float,
90  doc="Width of the region around detected sources to include in the DcrModel.",
91  default=3,
92  )
93  useModelWeights = pexConfig.Field(
94  dtype=bool,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=True,
97  )
98  splitSubfilters = pexConfig.Field(
99  dtype=bool,
100  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
101  "Instead of at the midpoint",
102  default=False,
103  )
104  regularizeModelIterations = pexConfig.Field(
105  dtype=float,
106  doc="Maximum relative change of the model allowed between iterations."
107  "Set to zero to disable.",
108  default=2.,
109  )
110  regularizeModelFrequency = pexConfig.Field(
111  dtype=float,
112  doc="Maximum relative change of the model allowed between subfilters."
113  "Set to zero to disable.",
114  default=4.,
115  )
116  convergenceMaskPlanes = pexConfig.ListField(
117  dtype=str,
118  default=["DETECTED"],
119  doc="Mask planes to use to calculate convergence."
120  )
121  regularizationWidth = pexConfig.Field(
122  dtype=int,
123  default=2,
124  doc="Minimum radius of a region to include in regularization, in pixels."
125  )
126  imageWarpMethod = pexConfig.Field(
127  dtype=str,
128  doc="Name of the warping kernel to use for shifting the image and variance planes.",
129  default="lanczos3",
130  )
131  maskWarpMethod = pexConfig.Field(
132  dtype=str,
133  doc="Name of the warping kernel to use for shifting the mask plane.",
134  default="bilinear",
135  )
136 
137  def setDefaults(self):
138  CompareWarpAssembleCoaddConfig.setDefaults(self)
139  self.doNImage = True
140  self.warpType = 'direct'
141  self.assembleStaticSkyModel.warpType = self.warpType
142  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
143  self.assembleStaticSkyModel.doNImage = False
144  self.assembleStaticSkyModel.doWrite = False
145  self.statistic = 'MEAN'
146 
147 
149  """Assemble DCR coadded images from a set of warps.
150 
151  Attributes
152  ----------
153  bufferSize : `int`
154  The number of pixels to grow each subregion by to allow for DCR.
155  warpCtrl : `lsst.afw.math.WarpingControl`
156  Configuration settings for warping an image
157 
158  Notes
159  -----
160  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
161  Warps (also called coadded temporary exposures), including the effects of
162  Differential Chromatic Refraction (DCR).
163  For full details of the mathematics and algorithm, please see
164  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
165 
166  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
167  each subfilter used in the iterative calculation.
168  It begins by dividing the bandpass-defining filter into N equal bandwidth
169  "subfilters", and divides the flux in each pixel from an initial coadd
170  equally into each as a "dcrModel". Because the airmass and parallactic
171  angle of each individual exposure is known, we can calculate the shift
172  relative to the center of the band in each subfilter due to DCR. For each
173  exposure we apply this shift as a linear transformation to the dcrModels
174  and stack the results to produce a DCR-matched exposure. The matched
175  exposures are subtracted from the input exposures to produce a set of
176  residual images, and these residuals are reverse shifted for each
177  exposures' subfilters and stacked. The shifted and stacked residuals are
178  added to the dcrModels to produce a new estimate of the flux in each pixel
179  within each subfilter. The dcrModels are solved for iteratively, which
180  continues until the solution from a new iteration improves by less than
181  a set percentage, or a maximum number of iterations is reached.
182  Two forms of regularization are employed to reduce unphysical results.
183  First, the new solution is averaged with the solution from the previous
184  iteration, which mitigates oscillating solutions where the model
185  overshoots with alternating very high and low values.
186  Second, a common degeneracy when the data have a limited range of airmass or
187  parallactic angle values is for one subfilter to be fit with very low or
188  negative values, while another subfilter is fit with very high values. This
189  typically appears in the form of holes next to sources in one subfilter,
190  and corresponding extended wings in another. Because each subfilter has
191  a narrow bandwidth we assume that physical sources that are above the noise
192  level will not vary in flux by more than a factor of `frequencyClampFactor`
193  between subfilters, and pixels that have flux deviations larger than that
194  factor will have the excess flux distributed evenly among all subfilters.
195  """
196 
197  ConfigClass = DcrAssembleCoaddConfig
198  _DefaultName = "dcrAssembleCoadd"
199 
200  @pipeBase.timeMethod
201  def runDataRef(self, dataRef, selectDataList=[]):
202  """Assemble a coadd from a set of warps.
203 
204  Coadd a set of Warps. Compute weights to be applied to each Warp and
205  find scalings to match the photometric zeropoint to a reference Warp.
206  Assemble the Warps using run method.
207  Forward model chromatic effects across multiple subfilters,
208  and subtract from the input Warps to build sets of residuals.
209  Use the residuals to construct a new ``DcrModel`` for each subfilter,
210  and iterate until the model converges.
211  Interpolate over NaNs and optionally write the coadd to disk.
212  Return the coadded exposure.
213 
214  Parameters
215  ----------
216  dataRef : `lsst.daf.persistence.ButlerDataRef`
217  Data reference defining the patch for coaddition and the
218  reference Warp
219  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
220  List of data references to warps. Data to be coadded will be
221  selected from this list based on overlap with the patch defined by
222  the data reference.
223 
224  Returns
225  -------
226  results : `lsst.pipe.base.Struct`
227  The Struct contains the following fields:
228 
229  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
230  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
231  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
232  - ``dcrNImages``: `list` of exposure count images for each subfilter
233  """
234  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
235  for subfilter in range(self.config.dcrNumSubfilters):
236  self.processResults(results.dcrCoadds[subfilter], dataRef)
237  if self.config.doWrite:
238  self.log.info("Persisting dcrCoadd")
239  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
240  numSubfilters=self.config.dcrNumSubfilters)
241  if self.config.doNImage and results.dcrNImages is not None:
242  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
243  numSubfilters=self.config.dcrNumSubfilters)
244 
245  return results
246 
247  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
248  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
249 
250  Sets the properties ``warpCtrl`` and ``bufferSize``.
251 
252  Parameters
253  ----------
254  templateCoadd : `lsst.afw.image.ExposureF`
255  The initial coadd exposure before accounting for DCR.
256  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
257  The data references to the input warped exposures.
258  weightList : `list` of `float`
259  The weight to give each input exposure in the coadd
260  Will be modified in place if ``doAirmassWeight`` is set.
261 
262  Returns
263  -------
264  dcrModels : `lsst.pipe.tasks.DcrModel`
265  Best fit model of the true sky after correcting chromatic effects.
266 
267  Raises
268  ------
269  NotImplementedError
270  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
271  """
272  filterInfo = templateCoadd.getFilter()
273  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
274  raise NotImplementedError("No minimum/maximum wavelength information found"
275  " in the filter definition! Please add lambdaMin and lambdaMax"
276  " to the Mapper class in your obs package.")
277  tempExpName = self.getTempExpDatasetName(self.warpType)
278  dcrShifts = []
279  for visitNum, tempExpRef in enumerate(tempExpRefList):
280  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
281  airmass = visitInfo.getBoresightAirmass()
282  if self.config.doAirmassWeight:
283  weightList[visitNum] *= airmass
284  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
285  filterInfo, self.config.dcrNumSubfilters))))
286  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
287  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
288  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
289  warpCache = 0
290  warpInterpLength = max(self.config.subregionSize)
291  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
292  self.config.maskWarpMethod,
293  cacheSize=warpCache, interpLength=warpInterpLength)
294  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
295  self.config.dcrNumSubfilters,
296  filterInfo=filterInfo,
297  psf=templateCoadd.getPsf())
298  return dcrModels
299 
300  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
301  supplementaryData=None):
302  """Assemble the coadd.
303 
304  Requires additional inputs Struct ``supplementaryData`` to contain a
305  ``templateCoadd`` that serves as the model of the static sky.
306 
307  Find artifacts and apply them to the warps' masks creating a list of
308  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
309  Then pass these alternative masks to the base class's assemble method.
310 
311  Divide the ``templateCoadd`` evenly between each subfilter of a
312  ``DcrModel`` as the starting best estimate of the true wavelength-
313  dependent sky. Forward model the ``DcrModel`` using the known
314  chromatic effects in each subfilter and calculate a convergence metric
315  based on how well the modeled template matches the input warps. If
316  the convergence has not yet reached the desired threshold, then shift
317  and stack the residual images to build a new ``DcrModel``. Apply
318  conditioning to prevent oscillating solutions between iterations or
319  between subfilters.
320 
321  Once the ``DcrModel`` reaches convergence or the maximum number of
322  iterations has been reached, fill the metadata for each subfilter
323  image and make them proper ``coaddExposure``s.
324 
325  Parameters
326  ----------
327  skyInfo : `lsst.pipe.base.Struct`
328  Patch geometry information, from getSkyInfo
329  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
330  The data references to the input warped exposures.
331  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
332  The image scalars correct for the zero point of the exposures.
333  weightList : `list` of `float`
334  The weight to give each input exposure in the coadd
335  supplementaryData : `lsst.pipe.base.Struct`
336  Result struct returned by ``makeSupplementaryData`` with components:
337 
338  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
339 
340  Returns
341  -------
342  result : `lsst.pipe.base.Struct`
343  Result struct with components:
344 
345  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
346  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
347  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
348  - ``dcrNImages``: `list` of exposure count images for each subfilter
349  """
350  templateCoadd = supplementaryData.templateCoadd
351  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
352  badMaskPlanes = self.config.badMaskPlanes[:]
353  badMaskPlanes.append("CLIPPED")
354  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
355  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
356  # Psf-Matching moves the real edge inwards
357  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
358 
359  stats = self.prepareStats(mask=badPixelMask)
360  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
361  if self.config.doNImage:
362  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
363  tempExpRefList, spanSetMaskList, stats.ctrl)
364  nImage = afwImage.ImageU(skyInfo.bbox)
365  # Note that this nImage will be a factor of dcrNumSubfilters higher than
366  # the nImage returned by assembleCoadd for most pixels. This is because each
367  # subfilter may have a different nImage, and fractional values are not allowed.
368  for dcrNImage in dcrNImages:
369  nImage += dcrNImage
370  else:
371  dcrNImages = None
372 
373  baseMask = templateCoadd.mask
374  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
375  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
376  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
377  subIter = 0
378  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
379  modelIter = 0
380  subIter += 1
381  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
382  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
383  dcrBBox = afwGeom.Box2I(subBBox)
384  dcrBBox.grow(self.bufferSize)
385  dcrBBox.clip(dcrModels.bbox)
386  if self.config.useModelWeights:
387  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
388  else:
389  modelWeights = 1.
390  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
391  imageScalerList, weightList, spanSetMaskList,
392  stats.ctrl)
393  self.log.info("Initial convergence : %s", convergenceMetric)
394  convergenceList = [convergenceMetric]
395  gainList = []
396  convergenceCheck = 1.
397  subfilterVariance = None
398  while (convergenceCheck > self.config.convergenceThreshold or
399  modelIter < self.config.minNumIter):
400  gain = self.calculateGain(convergenceList, gainList)
401  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
402  weightList, spanSetMaskList, stats.flags, stats.ctrl,
403  convergenceMetric, baseMask, subfilterVariance, gain,
404  modelWeights)
405  if self.config.useConvergence:
406  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
407  imageScalerList, weightList,
408  spanSetMaskList,
409  stats.ctrl)
410  if convergenceMetric == 0:
411  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
412  "most likely due to there being no valid data in the region.",
413  skyInfo.patchInfo.getIndex(), subIter)
414  break
415  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
416  if convergenceCheck < 0:
417  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
418  "iterations or desired convergence improvement of %s."
419  " Divergence: %s",
420  skyInfo.patchInfo.getIndex(), subIter,
421  self.config.convergenceThreshold, convergenceCheck)
422  break
423  convergenceList.append(convergenceMetric)
424  if modelIter > self.config.maxNumIter:
425  if self.config.useConvergence:
426  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
427  "before reaching desired convergence improvement of %s."
428  " Final convergence improvement: %s",
429  skyInfo.patchInfo.getIndex(), subIter,
430  self.config.convergenceThreshold, convergenceCheck)
431  break
432 
433  if self.config.useConvergence:
434  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
435  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
436  modelIter += 1
437  else:
438  if self.config.useConvergence:
439  self.log.info("Coadd patch %s subregion %s finished with "
440  "convergence metric %s after %s iterations",
441  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
442  else:
443  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
444  skyInfo.patchInfo.getIndex(), subIter, modelIter)
445  if self.config.useConvergence and convergenceMetric > 0:
446  self.log.info("Final convergence improvement was %.4f%% overall",
447  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
448 
449  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
450  calibration=self.scaleZeroPoint.getCalib(),
451  coaddInputs=self.inputRecorder.makeCoaddInputs(),
452  mask=templateCoadd.mask)
453  coaddExposure = self.stackCoadd(dcrCoadds)
454  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
455  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
456 
457  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
458  """Calculate the number of exposures contributing to each subfilter.
459 
460  Parameters
461  ----------
462  dcrModels : `lsst.pipe.tasks.DcrModel`
463  Best fit model of the true sky after correcting chromatic effects.
464  bbox : `lsst.afw.geom.box.Box2I`
465  Bounding box of the patch to coadd.
466  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
467  The data references to the input warped exposures.
468  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
469  Each element is dict with keys = mask plane name to add the spans to
470  statsCtrl : `lsst.afw.math.StatisticsControl`
471  Statistics control object for coadd
472 
473  Returns
474  -------
475  dcrNImages : `list` of `lsst.afw.image.ImageU`
476  List of exposure count images for each subfilter
477  """
478  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
479  tempExpName = self.getTempExpDatasetName(self.warpType)
480  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
481  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
482  visitInfo = exposure.getInfo().getVisitInfo()
483  wcs = exposure.getInfo().getWcs()
484  mask = exposure.mask
485  if altMaskSpans is not None:
486  self.applyAltMaskPlanes(mask, altMaskSpans)
487  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
488  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
489  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
490  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
491  return dcrNImages
492 
493  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
494  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
495  baseMask, subfilterVariance, gain, modelWeights):
496  """Assemble the DCR coadd for a sub-region.
497 
498  Build a DCR-matched template for each input exposure, then shift the
499  residuals according to the DCR in each subfilter.
500  Stack the shifted residuals and apply them as a correction to the
501  solution from the previous iteration.
502  Restrict the new model solutions from varying by more than a factor of
503  `modelClampFactor` from the last solution, and additionally restrict the
504  individual subfilter models from varying by more than a factor of
505  `frequencyClampFactor` from their average.
506  Finally, mitigate potentially oscillating solutions by averaging the new
507  solution with the solution from the previous iteration, weighted by
508  their convergence metric.
509 
510  Parameters
511  ----------
512  dcrModels : `lsst.pipe.tasks.DcrModel`
513  Best fit model of the true sky after correcting chromatic effects.
514  bbox : `lsst.afw.geom.box.Box2I`
515  Bounding box of the subregion to coadd.
516  dcrBBox :`lsst.afw.geom.box.Box2I`
517  Sub-region of the coadd which includes a buffer to allow for DCR.
518  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
519  The data references to the input warped exposures.
520  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
521  The image scalars correct for the zero point of the exposures.
522  weightList : `list` of `float`
523  The weight to give each input exposure in the coadd
524  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
525  Each element is dict with keys = mask plane name to add the spans to
526  statsFlags : `lsst.afw.math.Property`
527  Statistics settings for coaddition.
528  statsCtrl : `lsst.afw.math.StatisticsControl`
529  Statistics control object for coadd
530  convergenceMetric : `float`
531  Quality of fit metric for the matched templates of the input images.
532  baseMask : `lsst.afw.image.Mask`
533  Mask of the initial template coadd.
534  subfilterVariance : `list` of `numpy.ndarray`
535  The variance of each coadded subfilter image.
536  gain : `float`, optional
537  Relative weight to give the new solution when updating the model.
538  modelWeights : `numpy.ndarray` or `float`
539  A 2D array of weight values that tapers smoothly to zero away from detected sources.
540  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
541  """
542  tempExpName = self.getTempExpDatasetName(self.warpType)
543  residualGeneratorList = []
544 
545  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
546  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
547  visitInfo = exposure.getInfo().getVisitInfo()
548  wcs = exposure.getInfo().getWcs()
549  maskedImage = exposure.maskedImage
550  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
551  bbox=dcrBBox, wcs=wcs, mask=baseMask,
552  splitSubfilters=self.config.splitSubfilters)
553  imageScaler.scaleMaskedImage(maskedImage)
554  if altMaskSpans is not None:
555  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
556 
557  if self.config.removeMaskPlanes:
558  self.removeMaskPlanes(maskedImage)
559  maskedImage -= templateImage
560  maskedImage.image.array *= modelWeights
561  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
562  dcrModels.filter))
563 
564  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
565  statsFlags, statsCtrl, weightList,
566  mask=baseMask, gain=gain)
567  dcrModels.assign(dcrSubModelOut, bbox)
568 
569  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
570  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
571 
572  Parameters
573  ----------
574  residual : `lsst.afw.image.MaskedImageF`
575  The residual masked image for one exposure,
576  after subtracting the matched template
577  visitInfo : `lsst.afw.image.VisitInfo`
578  Metadata for the exposure.
579  bbox : `lsst.afw.geom.box.Box2I`
580  Sub-region of the coadd
581  wcs : `lsst.afw.geom.SkyWcs`
582  Coordinate system definition (wcs) for the exposure.
583  filterInfo : `lsst.afw.image.Filter`
584  The filter definition, set in the current instruments' obs package.
585  Required for any calculation of DCR, including making matched templates.
586 
587  Yields
588  ------
589  residualImage : `lsst.afw.image.maskedImageF`
590  The residual image for the next subfilter, shifted for DCR.
591  """
592  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
593  for dcr in dcrShift:
594  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
595 
596  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
597  statsFlags, statsCtrl, weightList,
598  mask, gain):
599  """Calculate a new DcrModel from a set of image residuals.
600 
601  Parameters
602  ----------
603  dcrModels : `lsst.pipe.tasks.DcrModel`
604  Current model of the true sky after correcting chromatic effects.
605  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
606  The residual image for the next subfilter, shifted for DCR.
607  bbox : `lsst.afw.geom.box.Box2I`
608  Sub-region of the coadd
609  statsFlags : `lsst.afw.math.Property`
610  Statistics settings for coaddition.
611  statsCtrl : `lsst.afw.math.StatisticsControl`
612  Statistics control object for coadd
613  weightList : `list` of `float`
614  The weight to give each input exposure in the coadd
615  mask : `lsst.afw.image.Mask`
616  Mask to use for each new model image.
617  gain : `float`
618  Relative weight to give the new solution when updating the model.
619 
620  Returns
621  -------
622  dcrModel : `lsst.pipe.tasks.DcrModel`
623  New model of the true sky after correcting chromatic effects.
624  """
625  maskMap = self.setRejectedMaskMapping(statsCtrl)
626  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
627  newModelImages = []
628  for subfilter, model in enumerate(dcrModels):
629  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
630  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
631  clipped, maskMap)
632  residual.setXY0(bbox.getBegin())
633  # `MaskedImage`s only support in-place addition, so rename for readability
634  residual += model[bbox]
635  newModel = residual
636  # Catch any invalid values
637  badPixels = ~np.isfinite(newModel.image.array)
638  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
639  # every iteration, masked regions will continually expand.
640  newModel.setMask(mask[bbox])
641  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
642  if self.config.regularizeModelIterations > 0:
643  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
644  self.config.regularizeModelIterations,
645  self.config.regularizationWidth)
646  newModelImages.append(newModel)
647  if self.config.regularizeModelFrequency > 0:
648  dcrModels.regularizeModelFreq(newModelImages, bbox,
649  self.config.regularizeModelFrequency,
650  self.config.regularizationWidth)
651  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
652  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
653 
654  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
655  weightList, spanSetMaskList, statsCtrl):
656  """Calculate a quality of fit metric for the matched templates.
657 
658  Parameters
659  ----------
660  dcrModels : `lsst.pipe.tasks.DcrModel`
661  Best fit model of the true sky after correcting chromatic effects.
662  bbox : `lsst.afw.geom.box.Box2I`
663  Sub-region to coadd
664  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
665  The data references to the input warped exposures.
666  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
667  The image scalars correct for the zero point of the exposures.
668  weightList : `list` of `float`
669  The weight to give each input exposure in the coadd
670  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
671  Each element is dict with keys = mask plane name to add the spans to
672  statsCtrl : `lsst.afw.math.StatisticsControl`
673  Statistics control object for coadd
674 
675  Returns
676  -------
677  convergenceMetric : `float`
678  Quality of fit metric for all input exposures, within the sub-region
679  """
680  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
681  nSigma = 3.
682  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
683  bufferSize=self.bufferSize)
684  tempExpName = self.getTempExpDatasetName(self.warpType)
685  weight = 0
686  metric = 0.
687  metricList = {}
688  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
689  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
690  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
691  imageScaler.scaleMaskedImage(exposure.maskedImage)
692  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
693  altMaskSpans=altMaskSpans)
694  metric += singleMetric*expWeight
695  metricList[tempExpRef.dataId["visit"]] = singleMetric
696  weight += expWeight
697  self.log.info("Individual metrics:\n%s", metricList)
698  return 1.0 if weight == 0.0 else metric/weight
699 
700  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
701  statsCtrl, altMaskSpans=None):
702  """Calculate a quality of fit metric for a single matched template.
703 
704  Parameters
705  ----------
706  dcrModels : `lsst.pipe.tasks.DcrModel`
707  Best fit model of the true sky after correcting chromatic effects.
708  exposure : `lsst.afw.image.ExposureF`
709  The input warped exposure to evaluate.
710  significanceImage : `numpy.ndarray`
711  Array of weights for each pixel corresponding to its significance
712  for the convergence calculation.
713  statsCtrl : `lsst.afw.math.StatisticsControl`
714  Statistics control object for coadd
715  altMaskSpans : `dict` containing spanSet lists, or None
716  The keys of the `dict` equal the mask plane name to add the spans to
717 
718  Returns
719  -------
720  convergenceMetric : `float`
721  Quality of fit metric for one exposure, within the sub-region.
722  """
723  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
724  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
725  visitInfo=exposure.getInfo().getVisitInfo(),
726  bbox=exposure.getBBox(),
727  wcs=exposure.getInfo().getWcs())
728  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
729  refVals = np.abs(templateImage.image.array)*significanceImage
730 
731  finitePixels = np.isfinite(diffVals)
732  if altMaskSpans is not None:
733  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
734  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
735  convergeMaskPixels = exposure.mask.array & convergeMask > 0
736  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
737  if np.sum(usePixels) == 0:
738  metric = 0.
739  else:
740  diffUse = diffVals[usePixels]
741  refUse = refVals[usePixels]
742  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
743  return metric
744 
745  def stackCoadd(self, dcrCoadds):
746  """Add a list of sub-band coadds together.
747 
748  Parameters
749  ----------
750  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
751  A list of coadd exposures, each exposure containing
752  the model for one subfilter.
753 
754  Returns
755  -------
756  coaddExposure : `lsst.afw.image.ExposureF`
757  A single coadd exposure that is the sum of the sub-bands.
758  """
759  coaddExposure = dcrCoadds[0].clone()
760  for coadd in dcrCoadds[1:]:
761  coaddExposure.maskedImage += coadd.maskedImage
762  return coaddExposure
763 
764  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
765  mask=None):
766  """Create a list of coadd exposures from a list of masked images.
767 
768  Parameters
769  ----------
770  dcrModels : `lsst.pipe.tasks.DcrModel`
771  Best fit model of the true sky after correcting chromatic effects.
772  skyInfo : `lsst.pipe.base.Struct`
773  Patch geometry information, from getSkyInfo
774  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
775  The data references to the input warped exposures.
776  weightList : `list` of `float`
777  The weight to give each input exposure in the coadd
778  calibration : `lsst.afw.Image.Calib`, optional
779  Scale factor to set the photometric zero point of an exposure.
780  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
781  A record of the observations that are included in the coadd.
782  mask : `lsst.afw.image.Mask`, optional
783  Optional mask to override the values in the final coadd.
784 
785  Returns
786  -------
787  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
788  A list of coadd exposures, each exposure containing
789  the model for one subfilter.
790  """
791  dcrCoadds = []
792  for model in dcrModels:
793  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
794  if calibration is not None:
795  coaddExposure.setCalib(calibration)
796  if coaddInputs is not None:
797  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
798  # Set the metadata for the coadd, including PSF and aperture corrections.
799  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
800  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
801  coaddExposure.setMaskedImage(model[skyInfo.bbox])
802  if mask is not None:
803  coaddExposure.setMask(mask)
804  dcrCoadds.append(coaddExposure)
805  return dcrCoadds
806 
807  def calculateGain(self, convergenceList, gainList):
808  """Calculate the gain to use for the current iteration.
809 
810  After calculating a new DcrModel, each value is averaged with the
811  value in the corresponding pixel from the previous iteration. This
812  reduces oscillating solutions that iterative techniques are plagued by,
813  and speeds convergence. By far the biggest changes to the model
814  happen in the first couple iterations, so we can also use a more
815  aggressive gain later when the model is changing slowly.
816 
817  Parameters
818  ----------
819  convergenceList : `list` of `float`
820  The quality of fit metric from each previous iteration.
821  gainList : `list` of `float`
822  The gains used in each previous iteration: appended with the new
823  gain value.
824  Gains are numbers between ``self.config.baseGain`` and 1.
825 
826  Returns
827  -------
828  gain : `float`
829  Relative weight to give the new solution when updating the model.
830  A value of 1.0 gives equal weight to both solutions.
831 
832  Raises
833  ------
834  ValueError
835  If ``len(convergenceList) != len(gainList)+1``.
836  """
837  nIter = len(convergenceList)
838  if nIter != len(gainList) + 1:
839  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
840  % (len(convergenceList), len(gainList)))
841 
842  if self.config.baseGain is None:
843  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
844  # The more subfilters being modeled, the lower the gain should be.
845  baseGain = 1./(self.config.dcrNumSubfilters - 1)
846  else:
847  baseGain = self.config.baseGain
848 
849  if self.config.useProgressiveGain and nIter > 2:
850  # To calculate the best gain to use, compare the past gains that have been used
851  # with the resulting convergences to estimate the best gain to use.
852  # Algorithmically, this is a Kalman filter.
853  # If forward modeling proceeds perfectly, the convergence metric should
854  # asymptotically approach a final value.
855  # We can estimate that value from the measured changes in convergence
856  # weighted by the gains used in each previous iteration.
857  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
858  for i in range(nIter - 1)]
859  # The convergence metric is strictly positive, so if the estimated final convergence is
860  # less than zero, force it to zero.
861  estFinalConv = np.array(estFinalConv)
862  estFinalConv[estFinalConv < 0] = 0
863  # Because the estimate may slowly change over time, only use the most recent measurements.
864  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
865  lastGain = gainList[-1]
866  lastConv = convergenceList[-2]
867  newConv = convergenceList[-1]
868  # The predicted convergence is the value we would get if the new model calculated
869  # in the previous iteration was perfect. Recall that the updated model that is
870  # actually used is the gain-weighted average of the new and old model,
871  # so the convergence would be similarly weighted.
872  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
873  # If the measured and predicted convergence are very close, that indicates
874  # that our forward model is accurate and we can use a more aggressive gain
875  # If the measured convergence is significantly worse (or better!) than predicted,
876  # that indicates that the model is not converging as expected and
877  # we should use a more conservative gain.
878  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
879  newGain = 1 - abs(delta)
880  # Average the gains to prevent oscillating solutions.
881  newGain = (newGain + lastGain)/2.
882  gain = max(baseGain, newGain)
883  else:
884  gain = baseGain
885  gainList.append(gain)
886  return gain
887 
888  def calculateModelWeights(self, dcrModels, dcrBBox):
889  """Build an array that smoothly tapers to 0 away from detected sources.
890 
891  Parameters
892  ----------
893  dcrModels : `lsst.pipe.tasks.DcrModel`
894  Best fit model of the true sky after correcting chromatic effects.
895  dcrBBox : `lsst.afw.geom.box.Box2I`
896  Sub-region of the coadd which includes a buffer to allow for DCR.
897 
898  Returns
899  -------
900  weights : `numpy.ndarray` or `float`
901  A 2D array of weight values that tapers smoothly to zero away from detected sources.
902  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
903 
904  Raises
905  ------
906  ValueError
907  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
908  """
909  if self.config.modelWeightsWidth < 0:
910  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
911  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
912  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
913  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
914  weights[convergeMaskPixels] = 1.
915  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
916  weights /= np.max(weights)
917  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:159
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)