lsst.pipe.tasks  16.0-29-gdc8abbdf+5
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 from scipy import ndimage
25 import lsst.afw.geom as afwGeom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.coadd.utils as coaddUtils
29 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
30 import lsst.pex.config as pexConfig
31 import lsst.pipe.base as pipeBase
32 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
33 
34 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
35 
36 
38  dcrNumSubfilters = pexConfig.Field(
39  dtype=int,
40  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
41  default=3,
42  )
43  maxNumIter = pexConfig.Field(
44  dtype=int,
45  doc="Maximum number of iterations of forward modeling.",
46  default=8,
47  )
48  minNumIter = pexConfig.Field(
49  dtype=int,
50  doc="Minimum number of iterations of forward modeling.",
51  default=3,
52  )
53  convergenceThreshold = pexConfig.Field(
54  dtype=float,
55  doc="Target relative change in convergence between iterations of forward modeling.",
56  default=0.001,
57  )
58  useConvergence = pexConfig.Field(
59  dtype=bool,
60  doc="Use convergence test as a forward modeling end condition?"
61  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
62  default=True,
63  )
64  baseGain = pexConfig.Field(
65  dtype=float,
66  doc="Relative weight to give the new solution when updating the model."
67  "A value of 1.0 gives equal weight to both solutions.",
68  default=1.,
69  )
70  useProgressiveGain = pexConfig.Field(
71  dtype=bool,
72  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence?",
73  default=True,
74  )
75  doAirmassWeight = pexConfig.Field(
76  dtype=bool,
77  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
78  default=True,
79  )
80  modelWeightsWidth = pexConfig.Field(
81  dtype=float,
82  doc="Width of the region around detected sources to include in the DcrModel.",
83  default=3,
84  )
85  useModelWeights = pexConfig.Field(
86  dtype=bool,
87  doc="Width of the region around detected sources to include in the DcrModel.",
88  default=True,
89  )
90  splitSubfilters = pexConfig.Field(
91  dtype=bool,
92  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
93  "Instead of at the midpoint",
94  default=False,
95  )
96  regularizeModelIterations = pexConfig.Field(
97  dtype=float,
98  doc="Maximum relative change of the model allowed between iterations."
99  "Set to zero to disable.",
100  default=2.,
101  )
102  regularizeModelFrequency = pexConfig.Field(
103  dtype=float,
104  doc="Maximum relative change of the model allowed between subfilters."
105  "Set to zero to disable.",
106  default=2.,
107  )
108  convergenceMaskPlanes = pexConfig.ListField(
109  dtype=str,
110  default=["DETECTED"],
111  doc="Mask planes to use to calculate convergence."
112  )
113  regularizationWidth = pexConfig.Field(
114  dtype=int,
115  default=2,
116  doc="Minimum radius of a region to include in regularization, in pixels."
117  )
118  imageWarpMethod = pexConfig.Field(
119  dtype=str,
120  doc="Name of the warping kernel to use for shifting the image and variance planes.",
121  default="lanczos3",
122  )
123  maskWarpMethod = pexConfig.Field(
124  dtype=str,
125  doc="Name of the warping kernel to use for shifting the mask plane.",
126  default="bilinear",
127  )
128 
129  def setDefaults(self):
130  CompareWarpAssembleCoaddConfig.setDefaults(self)
131  self.doNImage = True
132  self.warpType = 'direct'
133  self.assembleStaticSkyModel.warpType = self.warpType
134  self.assembleStaticSkyModel.doNImage = self.doNImage
135  self.statistic = 'MEAN'
136 
137 
139  """Assemble DCR coadded images from a set of warps.
140 
141  Notes
142  -----
143  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
144  Warps (also called coadded temporary exposures), including the effects of
145  Differential Chromatic Refraction (DCR).
146  For full details of the mathematics and algorithm, please see
147  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
148 
149  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
150  each subfilter used in the iterative calculation.
151  It begins by dividing the bandpass-defining filter into N equal bandwidth
152  "subfilters", and divides the flux in each pixel from an initial coadd
153  equally into each as a "dcrModel". Because the airmass and parallactic
154  angle of each individual exposure is known, we can calculate the shift
155  relative to the center of the band in each subfilter due to DCR. For each
156  exposure we apply this shift as a linear transformation to the dcrModels
157  and stack the results to produce a DCR-matched exposure. The matched
158  exposures are subtracted from the input exposures to produce a set of
159  residual images, and these residuals are reverse shifted for each
160  exposures' subfilters and stacked. The shifted and stacked residuals are
161  added to the dcrModels to produce a new estimate of the flux in each pixel
162  within each subfilter. The dcrModels are solved for iteratively, which
163  continues until the solution from a new iteration improves by less than
164  a set percentage, or a maximum number of iterations is reached.
165  Two forms of regularization are employed to reduce unphysical results.
166  First, the new solution is averaged with the solution from the previous
167  iteration, which mitigates oscillating solutions where the model
168  overshoots with alternating very high and low values.
169  Second, a common degeneracy when the data have a limited range of airmass or
170  parallactic angle values is for one subfilter to be fit with very low or
171  negative values, while another subfilter is fit with very high values. This
172  typically appears in the form of holes next to sources in one subfilter,
173  and corresponding extended wings in another. Because each subfilter has
174  a narrow bandwidth we assume that physical sources that are above the noise
175  level will not vary in flux by more than a factor of `frequencyClampFactor`
176  between subfilters, and pixels that have flux deviations larger than that
177  factor will have the excess flux distributed evenly among all subfilters.
178  """
179 
180  ConfigClass = DcrAssembleCoaddConfig
181  _DefaultName = "dcrAssembleCoadd"
182 
183  @pipeBase.timeMethod
184  def runDataRef(self, dataRef, selectDataList=[]):
185  """Assemble a coadd from a set of warps.
186 
187  Coadd a set of Warps. Compute weights to be applied to each Warp and
188  find scalings to match the photometric zeropoint to a reference Warp.
189  Assemble the Warps using run method.
190  Forward model chromatic effects across multiple subfilters,
191  and subtract from the input Warps to build sets of residuals.
192  Use the residuals to construct a new ``DcrModel`` for each subfilter,
193  and iterate until the model converges.
194  Interpolate over NaNs and optionally write the coadd to disk.
195  Return the coadded exposure.
196 
197  Parameters
198  ----------
199  dataRef : `lsst.daf.persistence.ButlerDataRef`
200  Data reference defining the patch for coaddition and the
201  reference Warp
202  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
203  List of data references to warps. Data to be coadded will be
204  selected from this list based on overlap with the patch defined by
205  the data reference.
206 
207  Returns
208  -------
209  results : `lsst.pipe.base.Struct`
210  The Struct contains the following fields:
211 
212  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
213  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
214  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
215  - ``dcrNImages``: `list` of exposure count images for each subfilter
216  """
217  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
218  for subfilter in range(self.config.dcrNumSubfilters):
219  self.processResults(results.dcrCoadds[subfilter], dataRef)
220  if self.config.doWrite:
221  self.log.info("Persisting dcrCoadd")
222  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
223  numSubfilters=self.config.dcrNumSubfilters)
224  if self.config.doNImage and results.dcrNImages is not None:
225  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
226  numSubfilters=self.config.dcrNumSubfilters)
227 
228  return results
229 
230  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
231  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
232 
233  Sets the properties ``filterInfo`` and ``bufferSize``.
234 
235  Parameters
236  ----------
237  templateCoadd : `lsst.afw.image.ExposureF`
238  The initial coadd exposure before accounting for DCR.
239  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
240  The data references to the input warped exposures.
241  weightList : `list` of `float`
242  The weight to give each input exposure in the coadd
243  Will be modified in place if ``doAirmassWeight`` is set.
244 
245  Returns
246  -------
247  dcrModels : `lsst.pipe.tasks.DcrModel`
248  Best fit model of the true sky after correcting chromatic effects.
249 
250  Raises
251  ------
252  NotImplementedError
253  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
254  """
255  filterInfo = templateCoadd.getFilter()
256  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
257  raise NotImplementedError("No minimum/maximum wavelength information found"
258  " in the filter definition! Please add lambdaMin and lambdaMax"
259  " to the Mapper class in your obs package.")
260  tempExpName = self.getTempExpDatasetName(self.warpType)
261  dcrShifts = []
262  for visitNum, tempExpRef in enumerate(tempExpRefList):
263  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
264  airmass = visitInfo.getBoresightAirmass()
265  if self.config.doAirmassWeight:
266  weightList[visitNum] *= airmass
267  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
268  filterInfo, self.config.dcrNumSubfilters))))
269  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
270  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
271  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
272  warpCache = 0
273  warpInterpLength = max(self.config.subregionSize)
274  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
275  self.config.maskWarpMethod,
276  cacheSize=warpCache, interpLength=warpInterpLength)
277  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
278  self.config.dcrNumSubfilters,
279  filterInfo=filterInfo,
280  psf=templateCoadd.getPsf())
281  return dcrModels
282 
283  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
284  supplementaryData=None):
285  """Assemble the coadd.
286 
287  Requires additional inputs Struct ``supplementaryData`` to contain a
288  ``templateCoadd`` that serves as the model of the static sky.
289 
290  Find artifacts and apply them to the warps' masks creating a list of
291  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
292  Then pass these alternative masks to the base class's assemble method.
293 
294  Divide the ``templateCoadd`` evenly between each subfilter of a
295  ``DcrModel`` as the starting best estimate of the true wavelength-
296  dependent sky. Forward model the ``DcrModel`` using the known
297  chromatic effects in each subfilter and calculate a convergence metric
298  based on how well the modeled template matches the input warps. If
299  the convergence has not yet reached the desired threshold, then shift
300  and stack the residual images to build a new ``DcrModel``. Apply
301  conditioning to prevent oscillating solutions between iterations or
302  between subfilters.
303 
304  Once the ``DcrModel`` reaches convergence or the maximum number of
305  iterations has been reached, fill the metadata for each subfilter
306  image and make them proper ``coaddExposure``s.
307 
308  Parameters
309  ----------
310  skyInfo : `lsst.pipe.base.Struct`
311  Patch geometry information, from getSkyInfo
312  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
313  The data references to the input warped exposures.
314  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
315  The image scalars correct for the zero point of the exposures.
316  weightList : `list` of `float`
317  The weight to give each input exposure in the coadd
318  supplementaryData : `lsst.pipe.base.Struct`
319  Result struct returned by ``makeSupplementaryData`` with components:
320 
321  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
322 
323  Returns
324  -------
325  result : `lsst.pipe.base.Struct`
326  Result struct with components:
327 
328  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
329  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
330  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
331  - ``dcrNImages``: `list` of exposure count images for each subfilter
332  """
333  templateCoadd = supplementaryData.templateCoadd
334  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
335  badMaskPlanes = self.config.badMaskPlanes[:]
336  badMaskPlanes.append("CLIPPED")
337  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
338  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
339  # Psf-Matching moves the real edge inwards
340  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
341 
342  stats = self.prepareStats(mask=badPixelMask)
343  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
344  if self.config.doNImage:
345  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
346  tempExpRefList, spanSetMaskList, stats.ctrl)
347  nImage = afwImage.ImageU(skyInfo.bbox)
348  # Note that this nImage will be a factor of dcrNumSubfilters higher than
349  # the nImage returned by assembleCoadd for most pixels. This is because each
350  # subfilter may have a different nImage, and fractional values are not allowed.
351  for dcrNImage in dcrNImages:
352  nImage += dcrNImage
353  else:
354  dcrNImages = None
355 
356  baseMask = templateCoadd.mask
357  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
358  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
359  modelIter = 0
360  self.log.info("Computing coadd over %s", subBBox)
361  if self.config.useModelWeights:
362  modelWeights = self.calculateModelWeights(templateCoadd.maskedImage[subBBox])
363  else:
364  return 1.
365  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
366  imageScalerList, weightList, spanSetMaskList,
367  stats.ctrl)
368  self.log.info("Initial convergence : %s", convergenceMetric)
369  convergenceList = [convergenceMetric]
370  convergenceCheck = 1.
371  subfilterVariance = None
372  while (convergenceCheck > self.config.convergenceThreshold or
373  modelIter < self.config.minNumIter):
374  gain = self.calculateGain(modelIter)
375  self.dcrAssembleSubregion(dcrModels, subBBox, tempExpRefList, imageScalerList,
376  weightList, spanSetMaskList, stats.flags, stats.ctrl,
377  convergenceMetric, baseMask, subfilterVariance, gain,
378  modelWeights)
379  if self.config.useConvergence:
380  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
381  imageScalerList, weightList,
382  spanSetMaskList,
383  stats.ctrl)
384  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
385  convergenceList.append(convergenceMetric)
386  if modelIter > self.config.maxNumIter:
387  if self.config.useConvergence:
388  self.log.warn("Coadd %s reached maximum iterations before reaching"
389  " desired convergence improvement of %s."
390  " Final convergence improvement: %s",
391  subBBox, self.config.convergenceThreshold, convergenceCheck)
392  break
393 
394  if self.config.useConvergence:
395  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.1f)",
396  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
397  modelIter += 1
398  else:
399  if self.config.useConvergence:
400  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
401  subBBox, convergenceMetric, modelIter)
402  else:
403  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
404  if self.config.useConvergence:
405  self.log.info("Final convergence improvement was %.4f%% overall",
406  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
407 
408  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
409  calibration=self.scaleZeroPoint.getCalib(),
410  coaddInputs=self.inputRecorder.makeCoaddInputs(),
411  mask=templateCoadd.mask)
412  coaddExposure = self.stackCoadd(dcrCoadds)
413  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
414  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
415 
416  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
417  """Calculate the number of exposures contributing to each subfilter.
418 
419  Parameters
420  ----------
421  dcrModels : `lsst.pipe.tasks.DcrModel`
422  Best fit model of the true sky after correcting chromatic effects.
423  bbox : `lsst.afw.geom.box.Box2I`
424  Bounding box of the patch to coadd.
425  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
426  The data references to the input warped exposures.
427  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
428  Each element is dict with keys = mask plane name to add the spans to
429  statsCtrl : `lsst.afw.math.StatisticsControl`
430  Statistics control object for coadd
431 
432  Returns
433  -------
434  dcrNImages : `list` of `lsst.afw.image.ImageU`
435  List of exposure count images for each subfilter
436  """
437  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
438  tempExpName = self.getTempExpDatasetName(self.warpType)
439  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
440  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
441  visitInfo = exposure.getInfo().getVisitInfo()
442  wcs = exposure.getInfo().getWcs()
443  mask = exposure.mask
444  if altMaskSpans is not None:
445  self.applyAltMaskPlanes(mask, altMaskSpans)
446  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
447  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
448  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
449  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
450  return dcrNImages
451 
452  def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList,
453  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
454  baseMask, subfilterVariance, gain, modelWeights):
455  """Assemble the DCR coadd for a sub-region.
456 
457  Build a DCR-matched template for each input exposure, then shift the
458  residuals according to the DCR in each subfilter.
459  Stack the shifted residuals and apply them as a correction to the
460  solution from the previous iteration.
461  Restrict the new model solutions from varying by more than a factor of
462  `modelClampFactor` from the last solution, and additionally restrict the
463  individual subfilter models from varying by more than a factor of
464  `frequencyClampFactor` from their average.
465  Finally, mitigate potentially oscillating solutions by averaging the new
466  solution with the solution from the previous iteration, weighted by
467  their convergence metric.
468 
469  Parameters
470  ----------
471  dcrModels : `lsst.pipe.tasks.DcrModel`
472  Best fit model of the true sky after correcting chromatic effects.
473  bbox : `lsst.afw.geom.box.Box2I`
474  Bounding box of the subregion to coadd.
475  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
476  The data references to the input warped exposures.
477  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
478  The image scalars correct for the zero point of the exposures.
479  weightList : `list` of `float`
480  The weight to give each input exposure in the coadd
481  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
482  Each element is dict with keys = mask plane name to add the spans to
483  statsFlags : `lsst.afw.math.Property`
484  Statistics settings for coaddition.
485  statsCtrl : `lsst.afw.math.StatisticsControl`
486  Statistics control object for coadd
487  convergenceMetric : `float`
488  Quality of fit metric for the matched templates of the input images.
489  baseMask : `lsst.afw.image.Mask`
490  Mask of the initial template coadd.
491  subfilterVariance : `list` of `numpy.ndarray`
492  The variance of each coadded subfilter image.
493  gain : `float`, optional
494  Relative weight to give the new solution when updating the model.
495  modelWeights : `numpy.ndarray` or `float`
496  A 2D array of weight values that tapers smoothly to zero away from detected sources.
497  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
498  """
499  bboxGrow = afwGeom.Box2I(bbox)
500  bboxGrow.grow(self.bufferSize)
501  bboxGrow.clip(dcrModels.bbox)
502 
503  tempExpName = self.getTempExpDatasetName(self.warpType)
504  residualGeneratorList = []
505 
506  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
507  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bboxGrow)
508  visitInfo = exposure.getInfo().getVisitInfo()
509  wcs = exposure.getInfo().getWcs()
510  maskedImage = exposure.maskedImage
511  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
512  bbox=bboxGrow, wcs=wcs, mask=baseMask,
513  splitSubfilters=self.config.splitSubfilters)
514  imageScaler.scaleMaskedImage(maskedImage)
515  if altMaskSpans is not None:
516  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
517 
518  if self.config.removeMaskPlanes:
519  self.removeMaskPlanes(maskedImage)
520  maskedImage -= templateImage
521  maskedImage.image.array *= modelWeights
522  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, bboxGrow, wcs,
523  dcrModels.filter))
524 
525  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, bboxGrow,
526  statsFlags, statsCtrl, weightList,
527  mask=baseMask, gain=gain)
528  dcrModels.assign(dcrSubModelOut, bbox)
529 
530  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
531  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
532 
533  Parameters
534  ----------
535  residual : `lsst.afw.image.MaskedImageF`
536  The residual masked image for one exposure,
537  after subtracting the matched template
538  visitInfo : `lsst.afw.image.VisitInfo`
539  Metadata for the exposure.
540  bbox : `lsst.afw.geom.box.Box2I`
541  Sub-region of the coadd
542  wcs : `lsst.afw.geom.SkyWcs`
543  Coordinate system definition (wcs) for the exposure.
544  filterInfo : `lsst.afw.image.Filter`
545  The filter definition, set in the current instruments' obs package.
546  Required for any calculation of DCR, including making matched templates.
547 
548  Yields
549  ------
550  residualImage : `lsst.afw.image.maskedImageF`
551  The residual image for the next subfilter, shifted for DCR.
552  """
553  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
554  for dcr in dcrShift:
555  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
556 
557  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
558  statsFlags, statsCtrl, weightList,
559  mask, gain):
560  """Calculate a new DcrModel from a set of image residuals.
561 
562  Parameters
563  ----------
564  dcrModels : `lsst.pipe.tasks.DcrModel`
565  Current model of the true sky after correcting chromatic effects.
566  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
567  The residual image for the next subfilter, shifted for DCR.
568  bbox : `lsst.afw.geom.box.Box2I`
569  Sub-region of the coadd
570  statsFlags : `lsst.afw.math.Property`
571  Statistics settings for coaddition.
572  statsCtrl : `lsst.afw.math.StatisticsControl`
573  Statistics control object for coadd
574  weightList : `list` of `float`
575  The weight to give each input exposure in the coadd
576  mask : `lsst.afw.image.Mask`
577  Mask to use for each new model image.
578  gain : `float`
579  Relative weight to give the new solution when updating the model.
580 
581  Returns
582  -------
583  dcrModel : `lsst.pipe.tasks.DcrModel`
584  New model of the true sky after correcting chromatic effects.
585  """
586  maskMap = self.setRejectedMaskMapping(statsCtrl)
587  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
588  newModelImages = []
589  for subfilter, model in enumerate(dcrModels):
590  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
591  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
592  clipped, maskMap)
593  residual.setXY0(bbox.getBegin())
594  # `MaskedImage`s only support in-place addition, so rename for readability
595  residual += model[bbox]
596  newModel = residual
597  # Catch any invalid values
598  badPixels = ~np.isfinite(newModel.image.array)
599  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
600  # every iteration, masked regions will continually expand.
601  newModel.setMask(mask[bbox])
602  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
603  if self.config.regularizeModelIterations > 0:
604  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
605  self.config.regularizeModelIterations,
606  self.config.regularizationWidth)
607  newModelImages.append(newModel)
608  if self.config.regularizeModelFrequency > 0:
609  dcrModels.regularizeModelFreq(newModelImages, bbox,
610  self.config.regularizeModelFrequency,
611  self.config.regularizationWidth)
612  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
613  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
614 
615  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
616  weightList, spanSetMaskList, statsCtrl):
617  """Calculate a quality of fit metric for the matched templates.
618 
619  Parameters
620  ----------
621  dcrModels : `lsst.pipe.tasks.DcrModel`
622  Best fit model of the true sky after correcting chromatic effects.
623  bbox : `lsst.afw.geom.box.Box2I`
624  Sub-region to coadd
625  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
626  The data references to the input warped exposures.
627  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
628  The image scalars correct for the zero point of the exposures.
629  weightList : `list` of `float`
630  The weight to give each input exposure in the coadd
631  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
632  Each element is dict with keys = mask plane name to add the spans to
633  statsCtrl : `lsst.afw.math.StatisticsControl`
634  Statistics control object for coadd
635 
636  Returns
637  -------
638  convergenceMetric : `float`
639  Quality of fit metric for all input exposures, within the sub-region
640  """
641  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
642  nSigma = 3.
643  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
644  bufferSize=self.bufferSize)
645  tempExpName = self.getTempExpDatasetName(self.warpType)
646  weight = 0
647  metric = 0.
648  metricList = {}
649  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
650  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
651  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
652  imageScaler.scaleMaskedImage(exposure.maskedImage)
653  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
654  altMaskSpans=altMaskSpans)
655  metric += singleMetric*expWeight
656  metricList[tempExpRef.dataId["visit"]] = singleMetric
657  weight += expWeight
658  self.log.info("Individual metrics:\n%s", metricList)
659  return 1.0 if weight == 0.0 else metric/weight
660 
661  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
662  statsCtrl, altMaskSpans=None):
663  """Calculate a quality of fit metric for a single matched template.
664 
665  Parameters
666  ----------
667  dcrModels : `lsst.pipe.tasks.DcrModel`
668  Best fit model of the true sky after correcting chromatic effects.
669  exposure : `lsst.afw.image.ExposureF`
670  The input warped exposure to evaluate.
671  significanceImage : `numpy.ndarray`
672  Array of weights for each pixel corresponding to its significance
673  for the convergence calculation.
674  statsCtrl : `lsst.afw.math.StatisticsControl`
675  Statistics control object for coadd
676  altMaskSpans : `dict` containing spanSet lists, or None
677  The keys of the `dict` equal the mask plane name to add the spans to
678 
679  Returns
680  -------
681  convergenceMetric : `float`
682  Quality of fit metric for one exposure, within the sub-region.
683  """
684  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
685  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
686  visitInfo=exposure.getInfo().getVisitInfo(),
687  bbox=exposure.getBBox(),
688  wcs=exposure.getInfo().getWcs())
689  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
690  refVals = np.abs(templateImage.image.array)*significanceImage
691 
692  finitePixels = np.isfinite(diffVals)
693  if altMaskSpans is not None:
694  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
695  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
696  convergeMaskPixels = exposure.mask.array & convergeMask > 0
697  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
698  if np.sum(usePixels) == 0:
699  metric = 0.
700  else:
701  diffUse = diffVals[usePixels]
702  refUse = refVals[usePixels]
703  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
704  return metric
705 
706  def stackCoadd(self, dcrCoadds):
707  """Add a list of sub-band coadds together.
708 
709  Parameters
710  ----------
711  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
712  A list of coadd exposures, each exposure containing
713  the model for one subfilter.
714 
715  Returns
716  -------
717  coaddExposure : `lsst.afw.image.ExposureF`
718  A single coadd exposure that is the sum of the sub-bands.
719  """
720  coaddExposure = dcrCoadds[0].clone()
721  for coadd in dcrCoadds[1:]:
722  coaddExposure.maskedImage += coadd.maskedImage
723  return coaddExposure
724 
725  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
726  mask=None):
727  """Create a list of coadd exposures from a list of masked images.
728 
729  Parameters
730  ----------
731  dcrModels : `lsst.pipe.tasks.DcrModel`
732  Best fit model of the true sky after correcting chromatic effects.
733  skyInfo : `lsst.pipe.base.Struct`
734  Patch geometry information, from getSkyInfo
735  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
736  The data references to the input warped exposures.
737  weightList : `list` of `float`
738  The weight to give each input exposure in the coadd
739  calibration : `lsst.afw.Image.Calib`, optional
740  Scale factor to set the photometric zero point of an exposure.
741  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
742  A record of the observations that are included in the coadd.
743  mask : `lsst.afw.image.Mask`, optional
744  Optional mask to override the values in the final coadd.
745 
746  Returns
747  -------
748  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
749  A list of coadd exposures, each exposure containing
750  the model for one subfilter.
751  """
752  dcrCoadds = []
753  for model in dcrModels:
754  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
755  if calibration is not None:
756  coaddExposure.setCalib(calibration)
757  if coaddInputs is not None:
758  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
759  # Set the metadata for the coadd, including PSF and aperture corrections.
760  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
761  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
762  coaddExposure.setMaskedImage(model[skyInfo.bbox])
763  if mask is not None:
764  coaddExposure.setMask(mask)
765  dcrCoadds.append(coaddExposure)
766  return dcrCoadds
767 
768  def calculateGain(self, modelIter):
769  """Calculate the gain to use for the current iteration.
770 
771  After calculating a new DcrModel, each value is averaged with the
772  value in the corresponding pixel from the previous iteration. This
773  reduces oscillating solutions that iterative techniques are plagued by,
774  and speeds convergence. By far the biggest changes to the model
775  happen in the first couple iterations, so we can also use a more
776  aggressive gain later when the model is changing slowly.
777 
778  Parameters
779  ----------
780  modelIter : `int`
781  The current iteration of forward modeling.
782 
783  Returns
784  -------
785  gain : `float`
786  Relative weight to give the new solution when updating the model.
787  A value of 1.0 gives equal weight to both solutions.
788  """
789  if self.config.useProgressiveGain:
790  iterGain = np.log(modelIter)*self.config.baseGain if modelIter > 0 else self.config.baseGain
791  return max(self.config.baseGain, iterGain)
792  return self.config.baseGain
793 
794  def calculateModelWeights(self, maskedImage):
795  """Build an array that smoothly tapers to 0 away from detected sources.
796 
797  Parameters
798  ----------
799  maskedImage : `numpy.ndarray`
800  The input masked image to calculate weights for.
801 
802  Returns
803  -------
804  weights : `numpy.ndarray` or `float`
805  A 2D array of weight values that tapers smoothly to zero away from detected sources.
806  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
807 
808  Raises
809  ------
810  ValueError
811  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
812  """
813  if self.config.modelWeightsWidth < 0:
814  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
815  convergeMask = maskedImage.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
816  convergeMaskPixels = maskedImage.mask.array & convergeMask > 0
817  weights = np.zeros_like(maskedImage.image.array)
818  weights[convergeMaskPixels] = 1.
819  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
820  weights /= np.max(weights)
821  return weights
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)