lsst.pipe.tasks  16.0-33-g83da4dfe+2
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from scipy import ndimage
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  doc="Maximum number of iterations of forward modeling.",
46  default=8,
47  )
48  minNumIter = pexConfig.Field(
49  dtype=int,
50  doc="Minimum number of iterations of forward modeling.",
51  default=3,
52  )
53  convergenceThreshold = pexConfig.Field(
54  dtype=float,
55  doc="Target relative change in convergence between iterations of forward modeling.",
56  default=0.001,
57  )
58  useConvergence = pexConfig.Field(
59  dtype=bool,
60  doc="Use convergence test as a forward modeling end condition?"
61  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
62  default=True,
63  )
64  baseGain = pexConfig.Field(
65  dtype=float,
66  doc="Relative weight to give the new solution when updating the model."
67  "A value of 1.0 gives equal weight to both solutions.",
68  default=1.,
69  )
70  useProgressiveGain = pexConfig.Field(
71  dtype=bool,
72  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence?",
73  default=True,
74  )
75  doAirmassWeight = pexConfig.Field(
76  dtype=bool,
77  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
78  default=True,
79  )
80  modelWeightsWidth = pexConfig.Field(
81  dtype=float,
82  doc="Width of the region around detected sources to include in the DcrModel.",
83  default=3,
84  )
85  useModelWeights = pexConfig.Field(
86  dtype=bool,
87  doc="Width of the region around detected sources to include in the DcrModel.",
88  default=True,
89  )
90  splitSubfilters = pexConfig.Field(
91  dtype=bool,
92  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
93  "Instead of at the midpoint",
94  default=False,
95  )
96  regularizeModelIterations = pexConfig.Field(
97  dtype=float,
98  doc="Maximum relative change of the model allowed between iterations."
99  "Set to zero to disable.",
100  default=2.,
101  )
102  regularizeModelFrequency = pexConfig.Field(
103  dtype=float,
104  doc="Maximum relative change of the model allowed between subfilters."
105  "Set to zero to disable.",
106  default=2.,
107  )
108  convergenceMaskPlanes = pexConfig.ListField(
109  dtype=str,
110  default=["DETECTED"],
111  doc="Mask planes to use to calculate convergence."
112  )
113  regularizationWidth = pexConfig.Field(
114  dtype=int,
115  default=2,
116  doc="Minimum radius of a region to include in regularization, in pixels."
117  )
118  imageWarpMethod = pexConfig.Field(
119  dtype=str,
120  doc="Name of the warping kernel to use for shifting the image and variance planes.",
121  default="lanczos3",
122  )
123  maskWarpMethod = pexConfig.Field(
124  dtype=str,
125  doc="Name of the warping kernel to use for shifting the mask plane.",
126  default="bilinear",
127  )
128 
129  def setDefaults(self):
130  CompareWarpAssembleCoaddConfig.setDefaults(self)
131  self.doNImage = True
132  self.warpType = 'direct'
133  self.assembleStaticSkyModel.warpType = self.warpType
134  self.assembleStaticSkyModel.doNImage = self.doNImage
135  self.statistic = 'MEAN'
136 
137 
139  """Assemble DCR coadded images from a set of warps.
140 
141  Attributes
142  ----------
143  bufferSize : `int`
144  The number of pixels to grow each subregion by to allow for DCR.
145  warpCtrl : `lsst.afw.math.WarpingControl`
146  Configuration settings for warping an image
147 
148  Notes
149  -----
150  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
151  Warps (also called coadded temporary exposures), including the effects of
152  Differential Chromatic Refraction (DCR).
153  For full details of the mathematics and algorithm, please see
154  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
155 
156  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
157  each subfilter used in the iterative calculation.
158  It begins by dividing the bandpass-defining filter into N equal bandwidth
159  "subfilters", and divides the flux in each pixel from an initial coadd
160  equally into each as a "dcrModel". Because the airmass and parallactic
161  angle of each individual exposure is known, we can calculate the shift
162  relative to the center of the band in each subfilter due to DCR. For each
163  exposure we apply this shift as a linear transformation to the dcrModels
164  and stack the results to produce a DCR-matched exposure. The matched
165  exposures are subtracted from the input exposures to produce a set of
166  residual images, and these residuals are reverse shifted for each
167  exposures' subfilters and stacked. The shifted and stacked residuals are
168  added to the dcrModels to produce a new estimate of the flux in each pixel
169  within each subfilter. The dcrModels are solved for iteratively, which
170  continues until the solution from a new iteration improves by less than
171  a set percentage, or a maximum number of iterations is reached.
172  Two forms of regularization are employed to reduce unphysical results.
173  First, the new solution is averaged with the solution from the previous
174  iteration, which mitigates oscillating solutions where the model
175  overshoots with alternating very high and low values.
176  Second, a common degeneracy when the data have a limited range of airmass or
177  parallactic angle values is for one subfilter to be fit with very low or
178  negative values, while another subfilter is fit with very high values. This
179  typically appears in the form of holes next to sources in one subfilter,
180  and corresponding extended wings in another. Because each subfilter has
181  a narrow bandwidth we assume that physical sources that are above the noise
182  level will not vary in flux by more than a factor of `frequencyClampFactor`
183  between subfilters, and pixels that have flux deviations larger than that
184  factor will have the excess flux distributed evenly among all subfilters.
185  """
186 
187  ConfigClass = DcrAssembleCoaddConfig
188  _DefaultName = "dcrAssembleCoadd"
189 
190  @pipeBase.timeMethod
191  def runDataRef(self, dataRef, selectDataList=[]):
192  """Assemble a coadd from a set of warps.
193 
194  Coadd a set of Warps. Compute weights to be applied to each Warp and
195  find scalings to match the photometric zeropoint to a reference Warp.
196  Assemble the Warps using run method.
197  Forward model chromatic effects across multiple subfilters,
198  and subtract from the input Warps to build sets of residuals.
199  Use the residuals to construct a new ``DcrModel`` for each subfilter,
200  and iterate until the model converges.
201  Interpolate over NaNs and optionally write the coadd to disk.
202  Return the coadded exposure.
203 
204  Parameters
205  ----------
206  dataRef : `lsst.daf.persistence.ButlerDataRef`
207  Data reference defining the patch for coaddition and the
208  reference Warp
209  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
210  List of data references to warps. Data to be coadded will be
211  selected from this list based on overlap with the patch defined by
212  the data reference.
213 
214  Returns
215  -------
216  results : `lsst.pipe.base.Struct`
217  The Struct contains the following fields:
218 
219  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
220  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
221  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
222  - ``dcrNImages``: `list` of exposure count images for each subfilter
223  """
224  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
225  for subfilter in range(self.config.dcrNumSubfilters):
226  self.processResults(results.dcrCoadds[subfilter], dataRef)
227  if self.config.doWrite:
228  self.log.info("Persisting dcrCoadd")
229  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
230  numSubfilters=self.config.dcrNumSubfilters)
231  if self.config.doNImage and results.dcrNImages is not None:
232  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
233  numSubfilters=self.config.dcrNumSubfilters)
234 
235  return results
236 
237  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
238  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
239 
240  Sets the properties ``warpCtrl`` and ``bufferSize``.
241 
242  Parameters
243  ----------
244  templateCoadd : `lsst.afw.image.ExposureF`
245  The initial coadd exposure before accounting for DCR.
246  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
247  The data references to the input warped exposures.
248  weightList : `list` of `float`
249  The weight to give each input exposure in the coadd
250  Will be modified in place if ``doAirmassWeight`` is set.
251 
252  Returns
253  -------
254  dcrModels : `lsst.pipe.tasks.DcrModel`
255  Best fit model of the true sky after correcting chromatic effects.
256 
257  Raises
258  ------
259  NotImplementedError
260  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
261  """
262  filterInfo = templateCoadd.getFilter()
263  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
264  raise NotImplementedError("No minimum/maximum wavelength information found"
265  " in the filter definition! Please add lambdaMin and lambdaMax"
266  " to the Mapper class in your obs package.")
267  tempExpName = self.getTempExpDatasetName(self.warpType)
268  dcrShifts = []
269  for visitNum, tempExpRef in enumerate(tempExpRefList):
270  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
271  airmass = visitInfo.getBoresightAirmass()
272  if self.config.doAirmassWeight:
273  weightList[visitNum] *= airmass
274  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
275  filterInfo, self.config.dcrNumSubfilters))))
276  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
277  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
278  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
279  warpCache = 0
280  warpInterpLength = max(self.config.subregionSize)
281  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
282  self.config.maskWarpMethod,
283  cacheSize=warpCache, interpLength=warpInterpLength)
284  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
285  self.config.dcrNumSubfilters,
286  filterInfo=filterInfo,
287  psf=templateCoadd.getPsf())
288  return dcrModels
289 
290  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
291  supplementaryData=None):
292  """Assemble the coadd.
293 
294  Requires additional inputs Struct ``supplementaryData`` to contain a
295  ``templateCoadd`` that serves as the model of the static sky.
296 
297  Find artifacts and apply them to the warps' masks creating a list of
298  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
299  Then pass these alternative masks to the base class's assemble method.
300 
301  Divide the ``templateCoadd`` evenly between each subfilter of a
302  ``DcrModel`` as the starting best estimate of the true wavelength-
303  dependent sky. Forward model the ``DcrModel`` using the known
304  chromatic effects in each subfilter and calculate a convergence metric
305  based on how well the modeled template matches the input warps. If
306  the convergence has not yet reached the desired threshold, then shift
307  and stack the residual images to build a new ``DcrModel``. Apply
308  conditioning to prevent oscillating solutions between iterations or
309  between subfilters.
310 
311  Once the ``DcrModel`` reaches convergence or the maximum number of
312  iterations has been reached, fill the metadata for each subfilter
313  image and make them proper ``coaddExposure``s.
314 
315  Parameters
316  ----------
317  skyInfo : `lsst.pipe.base.Struct`
318  Patch geometry information, from getSkyInfo
319  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
320  The data references to the input warped exposures.
321  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
322  The image scalars correct for the zero point of the exposures.
323  weightList : `list` of `float`
324  The weight to give each input exposure in the coadd
325  supplementaryData : `lsst.pipe.base.Struct`
326  Result struct returned by ``makeSupplementaryData`` with components:
327 
328  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
329 
330  Returns
331  -------
332  result : `lsst.pipe.base.Struct`
333  Result struct with components:
334 
335  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
336  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
337  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
338  - ``dcrNImages``: `list` of exposure count images for each subfilter
339  """
340  templateCoadd = supplementaryData.templateCoadd
341  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
342  badMaskPlanes = self.config.badMaskPlanes[:]
343  badMaskPlanes.append("CLIPPED")
344  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
345  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
346  # Psf-Matching moves the real edge inwards
347  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
348 
349  stats = self.prepareStats(mask=badPixelMask)
350  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
351  if self.config.doNImage:
352  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
353  tempExpRefList, spanSetMaskList, stats.ctrl)
354  nImage = afwImage.ImageU(skyInfo.bbox)
355  # Note that this nImage will be a factor of dcrNumSubfilters higher than
356  # the nImage returned by assembleCoadd for most pixels. This is because each
357  # subfilter may have a different nImage, and fractional values are not allowed.
358  for dcrNImage in dcrNImages:
359  nImage += dcrNImage
360  else:
361  dcrNImages = None
362 
363  baseMask = templateCoadd.mask
364  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
365  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
366  modelIter = 0
367  self.log.info("Computing coadd over %s", subBBox)
368  dcrBBox = afwGeom.Box2I(subBBox)
369  dcrBBox.grow(self.bufferSize)
370  dcrBBox.clip(dcrModels.bbox)
371  if self.config.useModelWeights:
372  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
373  else:
374  modelWeights = 1.
375  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
376  imageScalerList, weightList, spanSetMaskList,
377  stats.ctrl)
378  self.log.info("Initial convergence : %s", convergenceMetric)
379  convergenceList = [convergenceMetric]
380  convergenceCheck = 1.
381  subfilterVariance = None
382  while (convergenceCheck > self.config.convergenceThreshold or
383  modelIter < self.config.minNumIter):
384  gain = self.calculateGain(modelIter)
385  self.dcrAssembleSubregion(dcrModels, subBBox, dcrBBox, tempExpRefList, imageScalerList,
386  weightList, spanSetMaskList, stats.flags, stats.ctrl,
387  convergenceMetric, baseMask, subfilterVariance, gain,
388  modelWeights)
389  if self.config.useConvergence:
390  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
391  imageScalerList, weightList,
392  spanSetMaskList,
393  stats.ctrl)
394  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
395  convergenceList.append(convergenceMetric)
396  if modelIter > self.config.maxNumIter:
397  if self.config.useConvergence:
398  self.log.warn("Coadd %s reached maximum iterations before reaching"
399  " desired convergence improvement of %s."
400  " Final convergence improvement: %s",
401  subBBox, self.config.convergenceThreshold, convergenceCheck)
402  break
403 
404  if self.config.useConvergence:
405  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.1f)",
406  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
407  modelIter += 1
408  else:
409  if self.config.useConvergence:
410  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
411  subBBox, convergenceMetric, modelIter)
412  else:
413  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
414  if self.config.useConvergence:
415  self.log.info("Final convergence improvement was %.4f%% overall",
416  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
417 
418  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
419  calibration=self.scaleZeroPoint.getCalib(),
420  coaddInputs=self.inputRecorder.makeCoaddInputs(),
421  mask=templateCoadd.mask)
422  coaddExposure = self.stackCoadd(dcrCoadds)
423  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
424  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
425 
426  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
427  """Calculate the number of exposures contributing to each subfilter.
428 
429  Parameters
430  ----------
431  dcrModels : `lsst.pipe.tasks.DcrModel`
432  Best fit model of the true sky after correcting chromatic effects.
433  bbox : `lsst.afw.geom.box.Box2I`
434  Bounding box of the patch to coadd.
435  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
436  The data references to the input warped exposures.
437  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
438  Each element is dict with keys = mask plane name to add the spans to
439  statsCtrl : `lsst.afw.math.StatisticsControl`
440  Statistics control object for coadd
441 
442  Returns
443  -------
444  dcrNImages : `list` of `lsst.afw.image.ImageU`
445  List of exposure count images for each subfilter
446  """
447  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
448  tempExpName = self.getTempExpDatasetName(self.warpType)
449  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
450  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
451  visitInfo = exposure.getInfo().getVisitInfo()
452  wcs = exposure.getInfo().getWcs()
453  mask = exposure.mask
454  if altMaskSpans is not None:
455  self.applyAltMaskPlanes(mask, altMaskSpans)
456  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
457  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
458  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
459  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
460  return dcrNImages
461 
462  def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
463  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
464  baseMask, subfilterVariance, gain, modelWeights):
465  """Assemble the DCR coadd for a sub-region.
466 
467  Build a DCR-matched template for each input exposure, then shift the
468  residuals according to the DCR in each subfilter.
469  Stack the shifted residuals and apply them as a correction to the
470  solution from the previous iteration.
471  Restrict the new model solutions from varying by more than a factor of
472  `modelClampFactor` from the last solution, and additionally restrict the
473  individual subfilter models from varying by more than a factor of
474  `frequencyClampFactor` from their average.
475  Finally, mitigate potentially oscillating solutions by averaging the new
476  solution with the solution from the previous iteration, weighted by
477  their convergence metric.
478 
479  Parameters
480  ----------
481  dcrModels : `lsst.pipe.tasks.DcrModel`
482  Best fit model of the true sky after correcting chromatic effects.
483  bbox : `lsst.afw.geom.box.Box2I`
484  Bounding box of the subregion to coadd.
485  dcrBBox :`lsst.afw.geom.box.Box2I`
486  Sub-region of the coadd which includes a buffer to allow for DCR.
487  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
488  The data references to the input warped exposures.
489  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
490  The image scalars correct for the zero point of the exposures.
491  weightList : `list` of `float`
492  The weight to give each input exposure in the coadd
493  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
494  Each element is dict with keys = mask plane name to add the spans to
495  statsFlags : `lsst.afw.math.Property`
496  Statistics settings for coaddition.
497  statsCtrl : `lsst.afw.math.StatisticsControl`
498  Statistics control object for coadd
499  convergenceMetric : `float`
500  Quality of fit metric for the matched templates of the input images.
501  baseMask : `lsst.afw.image.Mask`
502  Mask of the initial template coadd.
503  subfilterVariance : `list` of `numpy.ndarray`
504  The variance of each coadded subfilter image.
505  gain : `float`, optional
506  Relative weight to give the new solution when updating the model.
507  modelWeights : `numpy.ndarray` or `float`
508  A 2D array of weight values that tapers smoothly to zero away from detected sources.
509  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
510  """
511  tempExpName = self.getTempExpDatasetName(self.warpType)
512  residualGeneratorList = []
513 
514  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
515  exposure = tempExpRef.get(tempExpName + "_sub", bbox=dcrBBox)
516  visitInfo = exposure.getInfo().getVisitInfo()
517  wcs = exposure.getInfo().getWcs()
518  maskedImage = exposure.maskedImage
519  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
520  bbox=dcrBBox, wcs=wcs, mask=baseMask,
521  splitSubfilters=self.config.splitSubfilters)
522  imageScaler.scaleMaskedImage(maskedImage)
523  if altMaskSpans is not None:
524  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
525 
526  if self.config.removeMaskPlanes:
527  self.removeMaskPlanes(maskedImage)
528  maskedImage -= templateImage
529  maskedImage.image.array *= modelWeights
530  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
531  dcrModels.filter))
532 
533  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox,
534  statsFlags, statsCtrl, weightList,
535  mask=baseMask, gain=gain)
536  dcrModels.assign(dcrSubModelOut, bbox)
537 
538  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
539  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
540 
541  Parameters
542  ----------
543  residual : `lsst.afw.image.MaskedImageF`
544  The residual masked image for one exposure,
545  after subtracting the matched template
546  visitInfo : `lsst.afw.image.VisitInfo`
547  Metadata for the exposure.
548  bbox : `lsst.afw.geom.box.Box2I`
549  Sub-region of the coadd
550  wcs : `lsst.afw.geom.SkyWcs`
551  Coordinate system definition (wcs) for the exposure.
552  filterInfo : `lsst.afw.image.Filter`
553  The filter definition, set in the current instruments' obs package.
554  Required for any calculation of DCR, including making matched templates.
555 
556  Yields
557  ------
558  residualImage : `lsst.afw.image.maskedImageF`
559  The residual image for the next subfilter, shifted for DCR.
560  """
561  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
562  for dcr in dcrShift:
563  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
564 
565  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
566  statsFlags, statsCtrl, weightList,
567  mask, gain):
568  """Calculate a new DcrModel from a set of image residuals.
569 
570  Parameters
571  ----------
572  dcrModels : `lsst.pipe.tasks.DcrModel`
573  Current model of the true sky after correcting chromatic effects.
574  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
575  The residual image for the next subfilter, shifted for DCR.
576  bbox : `lsst.afw.geom.box.Box2I`
577  Sub-region of the coadd
578  statsFlags : `lsst.afw.math.Property`
579  Statistics settings for coaddition.
580  statsCtrl : `lsst.afw.math.StatisticsControl`
581  Statistics control object for coadd
582  weightList : `list` of `float`
583  The weight to give each input exposure in the coadd
584  mask : `lsst.afw.image.Mask`
585  Mask to use for each new model image.
586  gain : `float`
587  Relative weight to give the new solution when updating the model.
588 
589  Returns
590  -------
591  dcrModel : `lsst.pipe.tasks.DcrModel`
592  New model of the true sky after correcting chromatic effects.
593  """
594  maskMap = self.setRejectedMaskMapping(statsCtrl)
595  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
596  newModelImages = []
597  for subfilter, model in enumerate(dcrModels):
598  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
599  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
600  clipped, maskMap)
601  residual.setXY0(bbox.getBegin())
602  # `MaskedImage`s only support in-place addition, so rename for readability
603  residual += model[bbox]
604  newModel = residual
605  # Catch any invalid values
606  badPixels = ~np.isfinite(newModel.image.array)
607  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
608  # every iteration, masked regions will continually expand.
609  newModel.setMask(mask[bbox])
610  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
611  if self.config.regularizeModelIterations > 0:
612  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
613  self.config.regularizeModelIterations,
614  self.config.regularizationWidth)
615  newModelImages.append(newModel)
616  if self.config.regularizeModelFrequency > 0:
617  dcrModels.regularizeModelFreq(newModelImages, bbox,
618  self.config.regularizeModelFrequency,
619  self.config.regularizationWidth)
620  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
621  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
622 
623  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
624  weightList, spanSetMaskList, statsCtrl):
625  """Calculate a quality of fit metric for the matched templates.
626 
627  Parameters
628  ----------
629  dcrModels : `lsst.pipe.tasks.DcrModel`
630  Best fit model of the true sky after correcting chromatic effects.
631  bbox : `lsst.afw.geom.box.Box2I`
632  Sub-region to coadd
633  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
634  The data references to the input warped exposures.
635  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
636  The image scalars correct for the zero point of the exposures.
637  weightList : `list` of `float`
638  The weight to give each input exposure in the coadd
639  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
640  Each element is dict with keys = mask plane name to add the spans to
641  statsCtrl : `lsst.afw.math.StatisticsControl`
642  Statistics control object for coadd
643 
644  Returns
645  -------
646  convergenceMetric : `float`
647  Quality of fit metric for all input exposures, within the sub-region
648  """
649  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
650  nSigma = 3.
651  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
652  bufferSize=self.bufferSize)
653  tempExpName = self.getTempExpDatasetName(self.warpType)
654  weight = 0
655  metric = 0.
656  metricList = {}
657  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
658  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
659  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
660  imageScaler.scaleMaskedImage(exposure.maskedImage)
661  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
662  altMaskSpans=altMaskSpans)
663  metric += singleMetric*expWeight
664  metricList[tempExpRef.dataId["visit"]] = singleMetric
665  weight += expWeight
666  self.log.info("Individual metrics:\n%s", metricList)
667  return 1.0 if weight == 0.0 else metric/weight
668 
669  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
670  statsCtrl, altMaskSpans=None):
671  """Calculate a quality of fit metric for a single matched template.
672 
673  Parameters
674  ----------
675  dcrModels : `lsst.pipe.tasks.DcrModel`
676  Best fit model of the true sky after correcting chromatic effects.
677  exposure : `lsst.afw.image.ExposureF`
678  The input warped exposure to evaluate.
679  significanceImage : `numpy.ndarray`
680  Array of weights for each pixel corresponding to its significance
681  for the convergence calculation.
682  statsCtrl : `lsst.afw.math.StatisticsControl`
683  Statistics control object for coadd
684  altMaskSpans : `dict` containing spanSet lists, or None
685  The keys of the `dict` equal the mask plane name to add the spans to
686 
687  Returns
688  -------
689  convergenceMetric : `float`
690  Quality of fit metric for one exposure, within the sub-region.
691  """
692  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
693  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
694  visitInfo=exposure.getInfo().getVisitInfo(),
695  bbox=exposure.getBBox(),
696  wcs=exposure.getInfo().getWcs())
697  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
698  refVals = np.abs(templateImage.image.array)*significanceImage
699 
700  finitePixels = np.isfinite(diffVals)
701  if altMaskSpans is not None:
702  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
703  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
704  convergeMaskPixels = exposure.mask.array & convergeMask > 0
705  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
706  if np.sum(usePixels) == 0:
707  metric = 0.
708  else:
709  diffUse = diffVals[usePixels]
710  refUse = refVals[usePixels]
711  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
712  return metric
713 
714  def stackCoadd(self, dcrCoadds):
715  """Add a list of sub-band coadds together.
716 
717  Parameters
718  ----------
719  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
720  A list of coadd exposures, each exposure containing
721  the model for one subfilter.
722 
723  Returns
724  -------
725  coaddExposure : `lsst.afw.image.ExposureF`
726  A single coadd exposure that is the sum of the sub-bands.
727  """
728  coaddExposure = dcrCoadds[0].clone()
729  for coadd in dcrCoadds[1:]:
730  coaddExposure.maskedImage += coadd.maskedImage
731  return coaddExposure
732 
733  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
734  mask=None):
735  """Create a list of coadd exposures from a list of masked images.
736 
737  Parameters
738  ----------
739  dcrModels : `lsst.pipe.tasks.DcrModel`
740  Best fit model of the true sky after correcting chromatic effects.
741  skyInfo : `lsst.pipe.base.Struct`
742  Patch geometry information, from getSkyInfo
743  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
744  The data references to the input warped exposures.
745  weightList : `list` of `float`
746  The weight to give each input exposure in the coadd
747  calibration : `lsst.afw.Image.Calib`, optional
748  Scale factor to set the photometric zero point of an exposure.
749  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
750  A record of the observations that are included in the coadd.
751  mask : `lsst.afw.image.Mask`, optional
752  Optional mask to override the values in the final coadd.
753 
754  Returns
755  -------
756  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
757  A list of coadd exposures, each exposure containing
758  the model for one subfilter.
759  """
760  dcrCoadds = []
761  for model in dcrModels:
762  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
763  if calibration is not None:
764  coaddExposure.setCalib(calibration)
765  if coaddInputs is not None:
766  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
767  # Set the metadata for the coadd, including PSF and aperture corrections.
768  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
769  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
770  coaddExposure.setMaskedImage(model[skyInfo.bbox])
771  if mask is not None:
772  coaddExposure.setMask(mask)
773  dcrCoadds.append(coaddExposure)
774  return dcrCoadds
775 
776  def calculateGain(self, modelIter):
777  """Calculate the gain to use for the current iteration.
778 
779  After calculating a new DcrModel, each value is averaged with the
780  value in the corresponding pixel from the previous iteration. This
781  reduces oscillating solutions that iterative techniques are plagued by,
782  and speeds convergence. By far the biggest changes to the model
783  happen in the first couple iterations, so we can also use a more
784  aggressive gain later when the model is changing slowly.
785 
786  Parameters
787  ----------
788  modelIter : `int`
789  The current iteration of forward modeling.
790 
791  Returns
792  -------
793  gain : `float`
794  Relative weight to give the new solution when updating the model.
795  A value of 1.0 gives equal weight to both solutions.
796  """
797  if self.config.useProgressiveGain:
798  iterGain = np.log(modelIter)*self.config.baseGain if modelIter > 0 else self.config.baseGain
799  return max(self.config.baseGain, iterGain)
800  return self.config.baseGain
801 
802  def calculateModelWeights(self, dcrModels, dcrBBox):
803  """Build an array that smoothly tapers to 0 away from detected sources.
804 
805  Parameters
806  ----------
807  dcrModels : `lsst.pipe.tasks.DcrModel`
808  Best fit model of the true sky after correcting chromatic effects.
809  dcrBBox : `lsst.afw.geom.box.Box2I`
810  Sub-region of the coadd which includes a buffer to allow for DCR.
811 
812  Returns
813  -------
814  weights : `numpy.ndarray` or `float`
815  A 2D array of weight values that tapers smoothly to zero away from detected sources.
816  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
817 
818  Raises
819  ------
820  ValueError
821  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
822  """
823  if self.config.modelWeightsWidth < 0:
824  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
825  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
826  convergeMaskPixels = dcrModels.mask.array & convergeMask > 0
827  weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
828  weights[convergeMaskPixels] = 1.
829  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
830  weights /= np.max(weights)
831  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)