lsst.pipe.tasks  16.0-23-ge8a9b866+2
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 import lsst.afw.geom as afwGeom
25 import lsst.afw.image as afwImage
26 import lsst.afw.math as afwMath
27 import lsst.coadd.utils as coaddUtils
28 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
32 
33 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
34 
35 
37  dcrNumSubfilters = pexConfig.Field(
38  dtype=int,
39  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
40  default=3,
41  )
42  maxNumIter = pexConfig.Field(
43  dtype=int,
44  doc="Maximum number of iterations of forward modeling.",
45  default=8,
46  )
47  minNumIter = pexConfig.Field(
48  dtype=int,
49  doc="Minimum number of iterations of forward modeling.",
50  default=3,
51  )
52  convergenceThreshold = pexConfig.Field(
53  dtype=float,
54  doc="Target relative change in convergence between iterations of forward modeling.",
55  default=0.001,
56  )
57  useConvergence = pexConfig.Field(
58  dtype=bool,
59  doc="Use convergence test as a forward modeling end condition?"
60  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
61  default=True,
62  )
63  baseGain = pexConfig.Field(
64  dtype=float,
65  doc="Relative weight to give the new solution when updating the model."
66  "A value of 1.0 gives equal weight to both solutions.",
67  default=1.,
68  )
69  useProgressiveGain = pexConfig.Field(
70  dtype=bool,
71  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence?",
72  default=True,
73  )
74  doAirmassWeight = pexConfig.Field(
75  dtype=bool,
76  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
77  default=True,
78  )
79  regularizeModelIterations = pexConfig.Field(
80  dtype=float,
81  doc="Maximum relative change of the model allowed between iterations."
82  "Set to zero to disable.",
83  default=2.,
84  )
85  regularizeModelFrequency = pexConfig.Field(
86  dtype=float,
87  doc="Maximum relative change of the model allowed between subfilters."
88  "Set to zero to disable.",
89  default=2.,
90  )
91  convergenceMaskPlanes = pexConfig.ListField(
92  dtype=str,
93  default=["DETECTED"],
94  doc="Mask planes to use to calculate convergence."
95  )
96  regularizationWidth = pexConfig.Field(
97  dtype=int,
98  default=2,
99  doc="Minimum radius of a region to include in regularization, in pixels."
100  )
101  imageWarpMethod = pexConfig.Field(
102  dtype=str,
103  doc="Name of the warping kernel to use for shifting the image and variance planes.",
104  default="lanczos3",
105  )
106  maskWarpMethod = pexConfig.Field(
107  dtype=str,
108  doc="Name of the warping kernel to use for shifting the mask plane.",
109  default="bilinear",
110  )
111 
112  def setDefaults(self):
113  CompareWarpAssembleCoaddConfig.setDefaults(self)
114  self.doNImage = True
115  self.warpType = 'direct'
116  self.assembleStaticSkyModel.warpType = self.warpType
117  self.assembleStaticSkyModel.doNImage = self.doNImage
118  self.statistic = 'MEAN'
119 
120 
122  """Assemble DCR coadded images from a set of warps.
123 
124  Notes
125  -----
126  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
127  Warps (also called coadded temporary exposures), including the effects of
128  Differential Chromatic Refraction (DCR).
129  For full details of the mathematics and algorithm, please see
130  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
131 
132  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
133  each subfilter used in the iterative calculation.
134  It begins by dividing the bandpass-defining filter into N equal bandwidth
135  "subfilters", and divides the flux in each pixel from an initial coadd
136  equally into each as a "dcrModel". Because the airmass and parallactic
137  angle of each individual exposure is known, we can calculate the shift
138  relative to the center of the band in each subfilter due to DCR. For each
139  exposure we apply this shift as a linear transformation to the dcrModels
140  and stack the results to produce a DCR-matched exposure. The matched
141  exposures are subtracted from the input exposures to produce a set of
142  residual images, and these residuals are reverse shifted for each
143  exposures' subfilters and stacked. The shifted and stacked residuals are
144  added to the dcrModels to produce a new estimate of the flux in each pixel
145  within each subfilter. The dcrModels are solved for iteratively, which
146  continues until the solution from a new iteration improves by less than
147  a set percentage, or a maximum number of iterations is reached.
148  Two forms of regularization are employed to reduce unphysical results.
149  First, the new solution is averaged with the solution from the previous
150  iteration, which mitigates oscillating solutions where the model
151  overshoots with alternating very high and low values.
152  Second, a common degeneracy when the data have a limited range of airmass or
153  parallactic angle values is for one subfilter to be fit with very low or
154  negative values, while another subfilter is fit with very high values. This
155  typically appears in the form of holes next to sources in one subfilter,
156  and corresponding extended wings in another. Because each subfilter has
157  a narrow bandwidth we assume that physical sources that are above the noise
158  level will not vary in flux by more than a factor of `frequencyClampFactor`
159  between subfilters, and pixels that have flux deviations larger than that
160  factor will have the excess flux distributed evenly among all subfilters.
161  """
162 
163  ConfigClass = DcrAssembleCoaddConfig
164  _DefaultName = "dcrAssembleCoadd"
165 
166  @pipeBase.timeMethod
167  def runDataRef(self, dataRef, selectDataList=[]):
168  """Assemble a coadd from a set of warps.
169 
170  Coadd a set of Warps. Compute weights to be applied to each Warp and
171  find scalings to match the photometric zeropoint to a reference Warp.
172  Assemble the Warps using run method.
173  Forward model chromatic effects across multiple subfilters,
174  and subtract from the input Warps to build sets of residuals.
175  Use the residuals to construct a new ``DcrModel`` for each subfilter,
176  and iterate until the model converges.
177  Interpolate over NaNs and optionally write the coadd to disk.
178  Return the coadded exposure.
179 
180  Parameters
181  ----------
182  dataRef : `lsst.daf.persistence.ButlerDataRef`
183  Data reference defining the patch for coaddition and the
184  reference Warp
185  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
186  List of data references to warps. Data to be coadded will be
187  selected from this list based on overlap with the patch defined by
188  the data reference.
189 
190  Returns
191  -------
192  results : `lsst.pipe.base.Struct`
193  The Struct contains the following fields:
194 
195  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
196  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
197  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
198  - ``dcrNImages``: `list` of exposure count images for each subfilter
199  """
200  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
201  for subfilter in range(self.config.dcrNumSubfilters):
202  self.processResults(results.dcrCoadds[subfilter], dataRef)
203  if self.config.doWrite:
204  self.log.info("Persisting dcrCoadd")
205  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
206  numSubfilters=self.config.dcrNumSubfilters)
207  if self.config.doNImage and results.dcrNImages is not None:
208  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
209  numSubfilters=self.config.dcrNumSubfilters)
210 
211  return results
212 
213  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
214  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
215 
216  Sets the properties ``filterInfo`` and ``bufferSize``.
217 
218  Parameters
219  ----------
220  templateCoadd : `lsst.afw.image.ExposureF`
221  The initial coadd exposure before accounting for DCR.
222  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
223  The data references to the input warped exposures.
224  weightList : `list` of `float`
225  The weight to give each input exposure in the coadd
226  Will be modified in place if ``doAirmassWeight`` is set.
227 
228  Returns
229  -------
230  dcrModels : `lsst.pipe.tasks.DcrModel`
231  Best fit model of the true sky after correcting chromatic effects.
232 
233  Raises
234  ------
235  NotImplementedError
236  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
237  """
238  filterInfo = templateCoadd.getFilter()
239  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
240  raise NotImplementedError("No minimum/maximum wavelength information found"
241  " in the filter definition! Please add lambdaMin and lambdaMax"
242  " to the Mapper class in your obs package.")
243  tempExpName = self.getTempExpDatasetName(self.warpType)
244  dcrShifts = []
245  for visitNum, tempExpRef in enumerate(tempExpRefList):
246  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
247  airmass = visitInfo.getBoresightAirmass()
248  if self.config.doAirmassWeight:
249  weightList[visitNum] *= airmass
250  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
251  filterInfo, self.config.dcrNumSubfilters))))
252  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
253  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
254  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
255  warpCache = 0
256  warpInterpLength = max(self.config.subregionSize)
257  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
258  self.config.maskWarpMethod,
259  cacheSize=warpCache, interpLength=warpInterpLength)
260  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
261  self.config.dcrNumSubfilters,
262  filterInfo=filterInfo,
263  psf=templateCoadd.getPsf())
264  return dcrModels
265 
266  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
267  supplementaryData=None):
268  """Assemble the coadd.
269 
270  Requires additional inputs Struct ``supplementaryData`` to contain a
271  ``templateCoadd`` that serves as the model of the static sky.
272 
273  Find artifacts and apply them to the warps' masks creating a list of
274  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
275  Then pass these alternative masks to the base class's assemble method.
276 
277  Divide the ``templateCoadd`` evenly between each subfilter of a
278  ``DcrModel`` as the starting best estimate of the true wavelength-
279  dependent sky. Forward model the ``DcrModel`` using the known
280  chromatic effects in each subfilter and calculate a convergence metric
281  based on how well the modeled template matches the input warps. If
282  the convergence has not yet reached the desired threshold, then shift
283  and stack the residual images to build a new ``DcrModel``. Apply
284  conditioning to prevent oscillating solutions between iterations or
285  between subfilters.
286 
287  Once the ``DcrModel`` reaches convergence or the maximum number of
288  iterations has been reached, fill the metadata for each subfilter
289  image and make them proper ``coaddExposure``s.
290 
291  Parameters
292  ----------
293  skyInfo : `lsst.pipe.base.Struct`
294  Patch geometry information, from getSkyInfo
295  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
296  The data references to the input warped exposures.
297  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
298  The image scalars correct for the zero point of the exposures.
299  weightList : `list` of `float`
300  The weight to give each input exposure in the coadd
301  supplementaryData : `lsst.pipe.base.Struct`
302  Result struct returned by ``makeSupplementaryData`` with components:
303 
304  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
305 
306  Returns
307  -------
308  result : `lsst.pipe.base.Struct`
309  Result struct with components:
310 
311  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
312  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
313  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
314  - ``dcrNImages``: `list` of exposure count images for each subfilter
315  """
316  templateCoadd = supplementaryData.templateCoadd
317  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
318  badMaskPlanes = self.config.badMaskPlanes[:]
319  badMaskPlanes.append("CLIPPED")
320  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
321  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
322  # Psf-Matching moves the real edge inwards
323  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
324 
325  stats = self.prepareStats(mask=badPixelMask)
326  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
327  if self.config.doNImage:
328  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
329  tempExpRefList, spanSetMaskList, stats.ctrl)
330  nImage = afwImage.ImageU(skyInfo.bbox)
331  # Note that this nImage will be a factor of dcrNumSubfilters higher than
332  # the nImage returned by assembleCoadd for most pixels. This is because each
333  # subfilter may have a different nImage, and fractional values are not allowed.
334  for dcrNImage in dcrNImages:
335  nImage += dcrNImage
336  else:
337  dcrNImages = None
338 
339  baseMask = templateCoadd.mask
340  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
341  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
342  modelIter = 0
343  self.log.info("Computing coadd over %s", subBBox)
344  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
345  imageScalerList, weightList, spanSetMaskList,
346  stats.ctrl)
347  self.log.info("Initial convergence : %s", convergenceMetric)
348  convergenceList = [convergenceMetric]
349  convergenceCheck = 1.
350  subfilterVariance = None
351  while (convergenceCheck > self.config.convergenceThreshold or
352  modelIter < self.config.minNumIter):
353  gain = self.calculateGain(modelIter, self.config.baseGain)
354  self.dcrAssembleSubregion(dcrModels, subBBox, tempExpRefList, imageScalerList,
355  weightList, spanSetMaskList, stats.flags, stats.ctrl,
356  convergenceMetric, baseMask, subfilterVariance, gain)
357  if self.config.useConvergence:
358  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
359  imageScalerList, weightList,
360  spanSetMaskList,
361  stats.ctrl)
362  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
363  convergenceList.append(convergenceMetric)
364  if modelIter > self.config.maxNumIter:
365  if self.config.useConvergence:
366  self.log.warn("Coadd %s reached maximum iterations before reaching"
367  " desired convergence improvement of %s."
368  " Final convergence improvement: %s",
369  subBBox, self.config.convergenceThreshold, convergenceCheck)
370  break
371 
372  if self.config.useConvergence:
373  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.1f)",
374  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
375  modelIter += 1
376  else:
377  if self.config.useConvergence:
378  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
379  subBBox, convergenceMetric, modelIter)
380  else:
381  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
382  if self.config.useConvergence:
383  self.log.info("Final convergence improvement was %.4f%% overall",
384  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
385 
386  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
387  calibration=self.scaleZeroPoint.getCalib(),
388  coaddInputs=self.inputRecorder.makeCoaddInputs(),
389  mask=templateCoadd.mask)
390  coaddExposure = self.stackCoadd(dcrCoadds)
391  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
392  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
393 
394  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
395  """Calculate the number of exposures contributing to each subfilter.
396 
397  Parameters
398  ----------
399  dcrModels : `lsst.pipe.tasks.DcrModel`
400  Best fit model of the true sky after correcting chromatic effects.
401  bbox : `lsst.afw.geom.box.Box2I`
402  Bounding box of the patch to coadd.
403  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
404  The data references to the input warped exposures.
405  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
406  Each element is dict with keys = mask plane name to add the spans to
407  statsCtrl : `lsst.afw.math.StatisticsControl`
408  Statistics control object for coadd
409 
410  Returns
411  -------
412  dcrNImages : `list` of `lsst.afw.image.ImageU`
413  List of exposure count images for each subfilter
414  """
415  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
416  tempExpName = self.getTempExpDatasetName(self.warpType)
417  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
418  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
419  visitInfo = exposure.getInfo().getVisitInfo()
420  wcs = exposure.getInfo().getWcs()
421  mask = exposure.mask
422  if altMaskSpans is not None:
423  self.applyAltMaskPlanes(mask, altMaskSpans)
424  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
425  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
426  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
427  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
428  return dcrNImages
429 
430  def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList,
431  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
432  baseMask, subfilterVariance, gain):
433  """Assemble the DCR coadd for a sub-region.
434 
435  Build a DCR-matched template for each input exposure, then shift the
436  residuals according to the DCR in each subfilter.
437  Stack the shifted residuals and apply them as a correction to the
438  solution from the previous iteration.
439  Restrict the new model solutions from varying by more than a factor of
440  `modelClampFactor` from the last solution, and additionally restrict the
441  individual subfilter models from varying by more than a factor of
442  `frequencyClampFactor` from their average.
443  Finally, mitigate potentially oscillating solutions by averaging the new
444  solution with the solution from the previous iteration, weighted by
445  their convergence metric.
446 
447  Parameters
448  ----------
449  dcrModels : `lsst.pipe.tasks.DcrModel`
450  Best fit model of the true sky after correcting chromatic effects.
451  bbox : `lsst.afw.geom.box.Box2I`
452  Bounding box of the subregion to coadd.
453  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
454  The data references to the input warped exposures.
455  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
456  The image scalars correct for the zero point of the exposures.
457  weightList : `list` of `float`
458  The weight to give each input exposure in the coadd
459  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
460  Each element is dict with keys = mask plane name to add the spans to
461  statsFlags : `lsst.afw.math.Property`
462  Statistics settings for coaddition.
463  statsCtrl : `lsst.afw.math.StatisticsControl`
464  Statistics control object for coadd
465  convergenceMetric : `float`
466  Quality of fit metric for the matched templates of the input images.
467  baseMask : `lsst.afw.image.Mask`
468  Mask of the initial template coadd.
469  subfilterVariance : `list` of `numpy.ndarray`
470  The variance of each coadded subfilter image.
471  gain : `float`, optional
472  Relative weight to give the new solution when updating the model.
473  """
474  bboxGrow = afwGeom.Box2I(bbox)
475  bboxGrow.grow(self.bufferSize)
476  bboxGrow.clip(dcrModels.bbox)
477 
478  tempExpName = self.getTempExpDatasetName(self.warpType)
479  residualGeneratorList = []
480 
481  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
482  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bboxGrow)
483  visitInfo = exposure.getInfo().getVisitInfo()
484  wcs = exposure.getInfo().getWcs()
485  maskedImage = exposure.maskedImage
486  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
487  bbox=bboxGrow, wcs=wcs, mask=baseMask)
488  imageScaler.scaleMaskedImage(maskedImage)
489  if altMaskSpans is not None:
490  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
491 
492  if self.config.removeMaskPlanes:
493  self.removeMaskPlanes(maskedImage)
494  maskedImage -= templateImage
495  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, bboxGrow, wcs,
496  dcrModels.filter))
497 
498  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, bboxGrow,
499  statsFlags, statsCtrl, weightList,
500  mask=baseMask, gain=gain)
501  dcrModels.assign(dcrSubModelOut, bbox)
502 
503  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
504  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
505 
506  Parameters
507  ----------
508  residual : `lsst.afw.image.MaskedImageF`
509  The residual masked image for one exposure,
510  after subtracting the matched template
511  visitInfo : `lsst.afw.image.VisitInfo`
512  Metadata for the exposure.
513  bbox : `lsst.afw.geom.box.Box2I`
514  Sub-region of the coadd
515  wcs : `lsst.afw.geom.SkyWcs`
516  Coordinate system definition (wcs) for the exposure.
517  filterInfo : `lsst.afw.image.Filter`
518  The filter definition, set in the current instruments' obs package.
519  Required for any calculation of DCR, including making matched templates.
520 
521  Yields
522  ------
523  residualImage : `lsst.afw.image.maskedImageF`
524  The residual image for the next subfilter, shifted for DCR.
525  """
526  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
527  for dcr in dcrShift:
528  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
529 
530  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox,
531  statsFlags, statsCtrl, weightList,
532  mask, gain):
533  """Calculate a new DcrModel from a set of image residuals.
534 
535  Parameters
536  ----------
537  dcrModels : `lsst.pipe.tasks.DcrModel`
538  Current model of the true sky after correcting chromatic effects.
539  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
540  The residual image for the next subfilter, shifted for DCR.
541  bbox : `lsst.afw.geom.box.Box2I`
542  Sub-region of the coadd
543  statsFlags : `lsst.afw.math.Property`
544  Statistics settings for coaddition.
545  statsCtrl : `lsst.afw.math.StatisticsControl`
546  Statistics control object for coadd
547  weightList : `list` of `float`
548  The weight to give each input exposure in the coadd
549  mask : `lsst.afw.image.Mask`
550  Mask to use for each new model image.
551  gain : `float`
552  Relative weight to give the new solution when updating the model.
553 
554  Returns
555  -------
556  dcrModel : `lsst.pipe.tasks.DcrModel`
557  New model of the true sky after correcting chromatic effects.
558  """
559  maskMap = self.setRejectedMaskMapping(statsCtrl)
560  clipped = dcrModels.mask.getPlaneBitMask("CLIPPED")
561  newModelImages = []
562  for subfilter, model in enumerate(dcrModels):
563  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
564  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
565  clipped, maskMap)
566  residual.setXY0(bbox.getBegin())
567  # `MaskedImage`s only support in-place addition, so rename for readability
568  residual += model[bbox]
569  newModel = residual
570  # Catch any invalid values
571  badPixels = ~np.isfinite(newModel.image.array)
572  # Overwrite the mask with one calculated previously. If the mask is allowed to adjust
573  # every iteration, masked regions will continually expand.
574  newModel.setMask(mask[bbox])
575  newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
576  if self.config.regularizeModelIterations > 0:
577  dcrModels.regularizeModelIter(subfilter, newModel, bbox,
578  self.config.regularizeModelIterations,
579  self.config.regularizationWidth)
580  newModelImages.append(newModel)
581  if self.config.regularizeModelFrequency > 0:
582  dcrModels.regularizeModelFreq(newModelImages, bbox,
583  self.config.regularizeModelFrequency,
584  self.config.regularizationWidth)
585  dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
586  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
587 
588  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
589  weightList, spanSetMaskList, statsCtrl):
590  """Calculate a quality of fit metric for the matched templates.
591 
592  Parameters
593  ----------
594  dcrModels : `lsst.pipe.tasks.DcrModel`
595  Best fit model of the true sky after correcting chromatic effects.
596  bbox : `lsst.afw.geom.box.Box2I`
597  Sub-region to coadd
598  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
599  The data references to the input warped exposures.
600  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
601  The image scalars correct for the zero point of the exposures.
602  weightList : `list` of `float`
603  The weight to give each input exposure in the coadd
604  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
605  Each element is dict with keys = mask plane name to add the spans to
606  statsCtrl : `lsst.afw.math.StatisticsControl`
607  Statistics control object for coadd
608 
609  Returns
610  -------
611  convergenceMetric : `float`
612  Quality of fit metric for all input exposures, within the sub-region
613  """
614  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
615  nSigma = 3.
616  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
617  bufferSize=self.bufferSize)
618  tempExpName = self.getTempExpDatasetName(self.warpType)
619  weight = 0
620  metric = 0.
621  metricList = {}
622  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
623  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
624  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
625  imageScaler.scaleMaskedImage(exposure.maskedImage)
626  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
627  altMaskSpans=altMaskSpans)
628  metric += singleMetric*expWeight
629  metricList[tempExpRef.dataId["visit"]] = singleMetric
630  weight += expWeight
631  self.log.info("Individual metrics:\n%s", metricList)
632  return 1.0 if weight == 0.0 else metric/weight
633 
634  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
635  statsCtrl, altMaskSpans=None):
636  """Calculate a quality of fit metric for a single matched template.
637 
638  Parameters
639  ----------
640  dcrModels : `lsst.pipe.tasks.DcrModel`
641  Best fit model of the true sky after correcting chromatic effects.
642  exposure : `lsst.afw.image.ExposureF`
643  The input warped exposure to evaluate.
644  significanceImage : `numpy.ndarray`
645  Array of weights for each pixel corresponding to its significance
646  for the convergence calculation.
647  statsCtrl : `lsst.afw.math.StatisticsControl`
648  Statistics control object for coadd
649  altMaskSpans : `dict` containing spanSet lists, or None
650  The keys of the `dict` equal the mask plane name to add the spans to
651 
652  Returns
653  -------
654  convergenceMetric : `float`
655  Quality of fit metric for one exposure, within the sub-region.
656  """
657  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
658  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
659  visitInfo=exposure.getInfo().getVisitInfo(),
660  bbox=exposure.getBBox(),
661  wcs=exposure.getInfo().getWcs())
662  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
663  refVals = np.abs(templateImage.image.array)*significanceImage
664 
665  finitePixels = np.isfinite(diffVals)
666  if altMaskSpans is not None:
667  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
668  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
669  convergeMaskPixels = exposure.mask.array & convergeMask > 0
670  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
671  if np.sum(usePixels) == 0:
672  metric = 0.
673  else:
674  diffUse = diffVals[usePixels]
675  refUse = refVals[usePixels]
676  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
677  return metric
678 
679  def stackCoadd(self, dcrCoadds):
680  """Add a list of sub-band coadds together.
681 
682  Parameters
683  ----------
684  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
685  A list of coadd exposures, each exposure containing
686  the model for one subfilter.
687 
688  Returns
689  -------
690  coaddExposure : `lsst.afw.image.ExposureF`
691  A single coadd exposure that is the sum of the sub-bands.
692  """
693  coaddExposure = dcrCoadds[0].clone()
694  for coadd in dcrCoadds[1:]:
695  coaddExposure.maskedImage += coadd.maskedImage
696  return coaddExposure
697 
698  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
699  mask=None):
700  """Create a list of coadd exposures from a list of masked images.
701 
702  Parameters
703  ----------
704  dcrModels : `lsst.pipe.tasks.DcrModel`
705  Best fit model of the true sky after correcting chromatic effects.
706  skyInfo : `lsst.pipe.base.Struct`
707  Patch geometry information, from getSkyInfo
708  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
709  The data references to the input warped exposures.
710  weightList : `list` of `float`
711  The weight to give each input exposure in the coadd
712  calibration : `lsst.afw.Image.Calib`, optional
713  Scale factor to set the photometric zero point of an exposure.
714  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
715  A record of the observations that are included in the coadd.
716  mask : `lsst.afw.image.Mask`, optional
717  Optional mask to override the values in the final coadd.
718 
719  Returns
720  -------
721  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
722  A list of coadd exposures, each exposure containing
723  the model for one subfilter.
724  """
725  dcrCoadds = []
726  for model in dcrModels:
727  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
728  if calibration is not None:
729  coaddExposure.setCalib(calibration)
730  if coaddInputs is not None:
731  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
732  # Set the metadata for the coadd, including PSF and aperture corrections.
733  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
734  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
735  coaddExposure.setMaskedImage(model[skyInfo.bbox])
736  if mask is not None:
737  coaddExposure.setMask(mask)
738  dcrCoadds.append(coaddExposure)
739  return dcrCoadds
740 
741  def calculateGain(self, modelIter, baseGain=1.):
742  """Calculate the gain to use for the current iteration.
743 
744  After calculating a new DcrModel, each value is averaged with the
745  value in the corresponding pixel from the previous iteration. This
746  reduces oscillating solutions that iterative techniques are plagued by,
747  and speeds convergence. By far the biggest changes to the model
748  happen in the first couple iterations, so we can also use a more
749  aggressive gain later when the model is changing slowly.
750 
751  Parameters
752  ----------
753  modelIter : `int`
754  The current iteration of forward modeling.
755  baseGain : `float`, optional
756  Description
757 
758  Returns
759  -------
760  gain : `float`
761  Relative weight to give the new solution when updating the model.
762  A value of 1.0 gives equal weight to both solutions.
763  """
764  if self.config.useProgressiveGain:
765  iterGain = np.log(modelIter) if modelIter > 0 else baseGain
766  return max(baseGain, iterGain)
767  return baseGain
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain)