lsst.pipe.tasks  16.0-5-g81851deb
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 import lsst.afw.geom as afwGeom
25 import lsst.afw.image as afwImage
26 import lsst.afw.math as afwMath
27 import lsst.coadd.utils as coaddUtils
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
31 from .dcrModel import applyDcr, calculateDcr, DcrModel
32 
33 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
34 
35 
37  dcrNumSubfilters = pexConfig.Field(
38  dtype=int,
39  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
40  default=3,
41  )
42  maxNumIter = pexConfig.Field(
43  dtype=int,
44  doc="Maximum number of iterations of forward modeling.",
45  default=8,
46  )
47  minNumIter = pexConfig.Field(
48  dtype=int,
49  doc="Minimum number of iterations of forward modeling.",
50  default=3,
51  )
52  convergenceThreshold = pexConfig.Field(
53  dtype=float,
54  doc="Target relative change in convergence between iterations of forward modeling.",
55  default=0.001,
56  )
57  useConvergence = pexConfig.Field(
58  dtype=bool,
59  doc="Use convergence test as a forward modeling end condition?"
60  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
61  default=True,
62  )
63  doAirmassWeight = pexConfig.Field(
64  dtype=bool,
65  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
66  default=True,
67  )
68  modelClampFactor = pexConfig.Field(
69  dtype=float,
70  doc="Maximum relative change of the model allowed between iterations.",
71  default=2.,
72  )
73  regularizeSigma = pexConfig.Field(
74  dtype=float,
75  doc="Threshold to exclude noise-like pixels from regularization.",
76  default=3.,
77  )
78  clampFrequency = pexConfig.Field(
79  dtype=float,
80  doc="Maximum relative change of the model allowed between subfilters.",
81  default=2.,
82  )
83  maxGain = pexConfig.Field(
84  dtype=float,
85  doc="Maximum convergence-weighted gain to apply between forward modeling iterations.",
86  default=2.,
87  )
88  minGain = pexConfig.Field(
89  dtype=float,
90  doc="Minimum convergence-weighted gain to apply between forward modeling iterations.",
91  default=0.5,
92  )
93  convergenceMaskPlanes = pexConfig.ListField(
94  dtype=str,
95  default=["DETECTED"],
96  doc="Mask planes to use to calculate convergence."
97  )
98  imageWarpMethod = pexConfig.Field(
99  dtype=str,
100  doc="Name of the warping kernel to use for shifting the image and variance planes.",
101  default="lanczos3",
102  )
103  maskWarpMethod = pexConfig.Field(
104  dtype=str,
105  doc="Name of the warping kernel to use for shifting the mask plane.",
106  default="bilinear",
107  )
108 
109  def setDefaults(self):
110  CompareWarpAssembleCoaddConfig.setDefaults(self)
111  self.doNImage = True
112  self.warpType = 'direct'
113  self.assembleStaticSkyModel.warpType = self.warpType
114  self.assembleStaticSkyModel.doNImage = self.doNImage
115  self.statistic = 'MEAN'
116 
117 
119  """Assemble DCR coadded images from a set of warps.
120 
121  Notes
122  -----
123  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
124  Warps (also called coadded temporary exposures), including the effects of
125  Differential Chromatic Refraction (DCR).
126  For full details of the mathematics and algorithm, please see
127  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
128 
129  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
130  each subfilter used in the iterative calculation.
131  It begins by dividing the bandpass-defining filter into N equal bandwidth
132  "subfilters", and divides the flux in each pixel from an initial coadd
133  equally into each as a "dcrModel". Because the airmass and parallactic
134  angle of each individual exposure is known, we can calculate the shift
135  relative to the center of the band in each subfilter due to DCR. For each
136  exposure we apply this shift as a linear transformation to the dcrModels
137  and stack the results to produce a DCR-matched exposure. The matched
138  exposures are subtracted from the input exposures to produce a set of
139  residual images, and these residuals are reverse shifted for each
140  exposures' subfilters and stacked. The shifted and stacked residuals are
141  added to the dcrModels to produce a new estimate of the flux in each pixel
142  within each subfilter. The dcrModels are solved for iteratively, which
143  continues until the solution from a new iteration improves by less than
144  a set percentage, or a maximum number of iterations is reached.
145  Two forms of regularization are employed to reduce unphysical results.
146  First, the new solution is averaged with the solution from the previous
147  iteration, which mitigates oscillating solutions where the model
148  overshoots with alternating very high and low values.
149  Second, a common degeneracy when the data have a limited range of airmass or
150  parallactic angle values is for one subfilter to be fit with very low or
151  negative values, while another subfilter is fit with very high values. This
152  typically appears in the form of holes next to sources in one subfilter,
153  and corresponding extended wings in another. Because each subfilter has
154  a narrow bandwidth we assume that physical sources that are above the noise
155  level will not vary in flux by more than a factor of `clampFrequency`
156  between subfilters, and pixels that have flux deviations larger than that
157  factor will have the excess flux distributed evenly among all subfilters.
158  """
159 
160  ConfigClass = DcrAssembleCoaddConfig
161  _DefaultName = "dcrAssembleCoadd"
162 
163  @pipeBase.timeMethod
164  def run(self, dataRef, selectDataList=[]):
165  """Assemble a coadd from a set of warps.
166 
167  Coadd a set of Warps. Compute weights to be applied to each Warp and
168  find scalings to match the photometric zeropoint to a reference Warp.
169  Assemble the Warps using assemble.
170  Forward model chromatic effects across multiple subfilters,
171  and subtract from the input Warps to build sets of residuals.
172  Use the residuals to construct a new ``DcrModel`` for each subfilter,
173  and iterate until the model converges.
174  Interpolate over NaNs and optionally write the coadd to disk.
175  Return the coadded exposure.
176 
177  Parameters
178  ----------
179  dataRef : `lsst.daf.persistence.ButlerDataRef`
180  Data reference defining the patch for coaddition and the
181  reference Warp
182  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
183  List of data references to warps. Data to be coadded will be
184  selected from this list based on overlap with the patch defined by
185  the data reference.
186 
187  Returns
188  -------
189  results : `lsst.pipe.base.Struct`
190  The Struct contains the following fields:
191 
192  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
193  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
194  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
195  - ``dcrNImages``: `list` of exposure count images for each subfilter
196  """
197  results = AssembleCoaddTask.run(self, dataRef, selectDataList=selectDataList)
198  for subfilter in range(self.config.dcrNumSubfilters):
199  self.processResults(results.dcrCoadds[subfilter], dataRef)
200  if self.config.doWrite:
201  self.log.info("Persisting dcrCoadd")
202  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
203  numSubfilters=self.config.dcrNumSubfilters)
204  if self.config.doNImage and results.dcrNImages is not None:
205  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
206  numSubfilters=self.config.dcrNumSubfilters)
207 
208  return results
209 
210  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
211  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
212 
213  Sets the properties ``filterInfo`` and ``bufferSize``.
214 
215  Parameters
216  ----------
217  templateCoadd : `lsst.afw.image.ExposureF`
218  The initial coadd exposure before accounting for DCR.
219  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
220  The data references to the input warped exposures.
221  weightList : `list` of `float`
222  The weight to give each input exposure in the coadd
223  Will be modified in place if ``doAirmassWeight`` is set.
224 
225  Returns
226  -------
227  dcrModels : `lsst.pipe.tasks.DcrModel`
228  Best fit model of the true sky after correcting chromatic effects.
229 
230  Raises
231  ------
232  NotImplementedError
233  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
234  """
235  self.filterInfo = templateCoadd.getFilter()
236  if np.isnan(self.filterInfo.getFilterProperty().getLambdaMin()):
237  raise NotImplementedError("No minimum/maximum wavelength information found"
238  " in the filter definition! Please add lambdaMin and lambdaMax"
239  " to the Mapper class in your obs package.")
240  tempExpName = self.getTempExpDatasetName(self.warpType)
241  dcrShifts = []
242  for visitNum, tempExpRef in enumerate(tempExpRefList):
243  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
244  airmass = visitInfo.getBoresightAirmass()
245  if self.config.doAirmassWeight:
246  weightList[visitNum] *= airmass
247  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
248  self.filterInfo, self.config.dcrNumSubfilters))))
249  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
250  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
251  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
252  warpCache = 0
253  warpInterpLength = max(self.config.subregionSize)
254  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
255  self.config.maskWarpMethod,
256  cacheSize=warpCache, interpLength=warpInterpLength)
257  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
258  self.config.dcrNumSubfilters,
259  self.filterInfo)
260  return dcrModels
261 
262  def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList,
263  supplementaryData=None):
264  """Assemble the coadd.
265 
266  Requires additional inputs Struct ``supplementaryData`` to contain a
267  ``templateCoadd`` that serves as the model of the static sky.
268 
269  Find artifacts and apply them to the warps' masks creating a list of
270  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
271  Then pass these alternative masks to the base class's assemble method.
272 
273  Divide the ``templateCoadd`` evenly between each subfilter of a
274  ``DcrModel`` as the starting best estimate of the true wavelength-
275  dependent sky. Forward model the ``DcrModel`` using the known
276  chromatic effects in each subfilter and calculate a convergence metric
277  based on how well the modeled template matches the input warps. If
278  the convergence has not yet reached the desired threshold, then shift
279  and stack the residual images to build a new ``DcrModel``. Apply
280  conditioning to prevent oscillating solutions between iterations or
281  between subfilters.
282 
283  Once the ``DcrModel`` reaches convergence or the maximum number of
284  iterations has been reached, fill the metadata for each subfilter
285  image and make them proper ``coaddExposure``s.
286 
287  Parameters
288  ----------
289  skyInfo : `lsst.pipe.base.Struct`
290  Patch geometry information, from getSkyInfo
291  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
292  The data references to the input warped exposures.
293  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
294  The image scalars correct for the zero point of the exposures.
295  weightList : `list` of `float`
296  The weight to give each input exposure in the coadd
297  supplementaryData : `lsst.pipe.base.Struct`
298  Result struct returned by ``makeSupplementaryData`` with components:
299 
300  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
301 
302  Returns
303  -------
304  result : `lsst.pipe.base.Struct`
305  Result struct with components:
306 
307  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
308  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
309  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
310  - ``dcrNImages``: `list` of exposure count images for each subfilter
311  """
312  templateCoadd = supplementaryData.templateCoadd
313  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
314  badMaskPlanes = self.config.badMaskPlanes[:]
315  badMaskPlanes.append("CLIPPED")
316  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
317  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
318  # Psf-Matching moves the real edge inwards
319  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
320 
321  stats = self.prepareStats(mask=badPixelMask)
322  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
323  if self.config.doNImage:
324  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
325  tempExpRefList, spanSetMaskList, stats.ctrl)
326  nImage = afwImage.ImageU(skyInfo.bbox)
327  # Note that this nImage will be a factor of dcrNumSubfilters higher than
328  # the nImage returned by assembleCoadd for most pixels. This is because each
329  # subfilter may have a different nImage, and fractional values are not allowed.
330  for dcrNImage in dcrNImages:
331  nImage += dcrNImage
332  else:
333  dcrNImages = None
334 
335  baseMask = templateCoadd.mask
336  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
337  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
338  modelIter = 0
339  self.log.info("Computing coadd over %s", subBBox)
340  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
341  imageScalerList, weightList, spanSetMaskList,
342  stats.ctrl)
343  self.log.info("Initial convergence : %s", convergenceMetric)
344  convergenceList = [convergenceMetric]
345  convergenceCheck = 1.
346  subfilterVariance = None
347  while (convergenceCheck > self.config.convergenceThreshold or
348  modelIter < self.config.minNumIter):
349  self.dcrAssembleSubregion(dcrModels, subBBox, tempExpRefList, imageScalerList,
350  weightList, spanSetMaskList, stats.flags, stats.ctrl,
351  convergenceMetric, baseMask, subfilterVariance)
352  if self.config.useConvergence:
353  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
354  imageScalerList, weightList,
355  spanSetMaskList,
356  stats.ctrl)
357  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
358  convergenceList.append(convergenceMetric)
359  if modelIter > self.config.maxNumIter:
360  if self.config.useConvergence:
361  self.log.warn("Coadd %s reached maximum iterations before reaching"
362  " desired convergence improvement of %s."
363  " Final convergence improvement: %s",
364  subBBox, self.config.convergenceThreshold, convergenceCheck)
365  break
366 
367  if self.config.useConvergence:
368  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement",
369  modelIter, convergenceMetric, 100.*convergenceCheck)
370  modelIter += 1
371  else:
372  if self.config.useConvergence:
373  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
374  subBBox, convergenceMetric, modelIter)
375  else:
376  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
377  if self.config.useConvergence:
378  self.log.info("Final convergence improvement was %.4f%% overall",
379  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
380  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
381  calibration=self.scaleZeroPoint.getCalib(),
382  coaddInputs=self.inputRecorder.makeCoaddInputs())
383  coaddExposure = self.stackCoadd(dcrCoadds)
384  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
385  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
386 
387  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
388  """Calculate the number of exposures contributing to each subfilter.
389 
390  Parameters
391  ----------
392  dcrModels : `lsst.pipe.tasks.DcrModel`
393  Best fit model of the true sky after correcting chromatic effects.
394  bbox : `lsst.afw.geom.box.Box2I`
395  Bounding box of the patch to coadd.
396  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
397  The data references to the input warped exposures.
398  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
399  Each element is dict with keys = mask plane name to add the spans to
400  statsCtrl : `lsst.afw.math.StatisticsControl`
401  Statistics control object for coadd
402 
403  Returns
404  -------
405  dcrNImages : `list` of `lsst.afw.image.ImageU`
406  List of exposure count images for each subfilter
407  """
408  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
409  tempExpName = self.getTempExpDatasetName(self.warpType)
410  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
411  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
412  visitInfo = exposure.getInfo().getVisitInfo()
413  wcs = exposure.getInfo().getWcs()
414  mask = exposure.mask
415  if altMaskSpans is not None:
416  self.applyAltMaskPlanes(mask, altMaskSpans)
417  dcrShift = calculateDcr(visitInfo, wcs, self.filterInfo, self.config.dcrNumSubfilters)
418  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
419  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
420  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
421  return dcrNImages
422 
423  def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList,
424  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
425  baseMask, subfilterVariance):
426  """Assemble the DCR coadd for a sub-region.
427 
428  Build a DCR-matched template for each input exposure, then shift the
429  residuals according to the DCR in each subfilter.
430  Stack the shifted residuals and apply them as a correction to the
431  solution from the previous iteration.
432  Restrict the new model solutions from varying by more than a factor of
433  `modelClampFactor` from the last solution, and additionally restrict the
434  individual subfilter models from varying by more than a factor of
435  `clampFrequency` from their average.
436  Finally, mitigate potentially oscillating solutions by averaging the new
437  solution with the solution from the previous iteration, weighted by
438  their convergence metric.
439 
440  Parameters
441  ----------
442  dcrModels : `lsst.pipe.tasks.DcrModel`
443  Best fit model of the true sky after correcting chromatic effects.
444  bbox : `lsst.afw.geom.box.Box2I`
445  Bounding box of the subregion to coadd.
446  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
447  The data references to the input warped exposures.
448  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
449  The image scalars correct for the zero point of the exposures.
450  weightList : `list` of `float`
451  The weight to give each input exposure in the coadd
452  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
453  Each element is dict with keys = mask plane name to add the spans to
454  statsFlags : `lsst.afw.math.Property`
455  Statistics settings for coaddition.
456  statsCtrl : `lsst.afw.math.StatisticsControl`
457  Statistics control object for coadd
458  convergenceMetric : `float`
459  Quality of fit metric for the matched templates of the input images.
460  baseMask : `lsst.afw.image.Mask`
461  Mask of the initial template coadd.
462  subfilterVariance : `list` of `numpy.ndarray`
463  The variance of each coadded subfilter image.
464  """
465  bboxGrow = afwGeom.Box2I(bbox)
466  bboxGrow.grow(self.bufferSize)
467  bboxGrow.clip(dcrModels.getBBox())
468 
469  tempExpName = self.getTempExpDatasetName(self.warpType)
470  residualGeneratorList = []
471 
472  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
473  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bboxGrow)
474  visitInfo = exposure.getInfo().getVisitInfo()
475  wcs = exposure.getInfo().getWcs()
476  maskedImage = exposure.maskedImage
477  templateImage = dcrModels.buildMatchedTemplate(self.warpCtrl, visitInfo=visitInfo, bbox=bboxGrow,
478  wcs=wcs, mask=baseMask)
479  imageScaler.scaleMaskedImage(maskedImage)
480  if altMaskSpans is not None:
481  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
482 
483  if self.config.removeMaskPlanes:
484  self.removeMaskPlanes(maskedImage)
485  maskedImage -= templateImage
486  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, bboxGrow, wcs))
487 
488  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, bboxGrow,
489  statsFlags, statsCtrl, weightList)
490  dcrSubModelOut.regularizeModel(bboxGrow, baseMask, statsCtrl, self.config.regularizeSigma,
491  self.config.clampFrequency, self.config.convergenceMaskPlanes)
492  dcrModels.assign(dcrSubModelOut, bbox)
493 
494  def dcrResiduals(self, residual, visitInfo, bbox, wcs):
495  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
496 
497  Parameters
498  ----------
499  residual : `lsst.afw.image.MaskedImageF`
500  The residual masked image for one exposure,
501  after subtracting the matched template
502  visitInfo : `lsst.afw.image.VisitInfo`
503  Metadata for the exposure.
504  bbox : `lsst.afw.geom.box.Box2I`
505  Sub-region of the coadd
506  wcs : `lsst.afw.geom.SkyWcs`
507  Coordinate system definition (wcs) for the exposure.
508 
509  Yields
510  ------
511  residualImage : `lsst.afw.image.maskedImageF`
512  The residual image for the next subfilter, shifted for DCR.
513  """
514  dcrShift = calculateDcr(visitInfo, wcs, self.filterInfo, self.config.dcrNumSubfilters)
515  for dcr in dcrShift:
516  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
517 
518  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList):
519  """Calculate a new DcrModel from a set of image residuals.
520 
521  Parameters
522  ----------
523  dcrModels : `lsst.pipe.tasks.DcrModel`
524  Current model of the true sky after correcting chromatic effects.
525  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
526  The residual image for the next subfilter, shifted for DCR.
527  bbox : `lsst.afw.geom.box.Box2I`
528  Sub-region of the coadd
529  statsFlags : `lsst.afw.math.Property`
530  Statistics settings for coaddition.
531  statsCtrl : `lsst.afw.math.StatisticsControl`
532  Statistics control object for coadd
533  weightList : `list` of `float`
534  The weight to give each input exposure in the coadd
535 
536  Returns
537  -------
538  dcrModel : `lsst.pipe.tasks.DcrModel`
539  New model of the true sky after correcting chromatic effects.
540  """
541  maskMap = self.setRejectedMaskMapping(statsCtrl)
542  clipped = dcrModels[0].mask.getPlaneBitMask("CLIPPED")
543  newModelImages = []
544  for subfilter, model in enumerate(dcrModels):
545  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
546  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
547  clipped, maskMap)
548  residual.setXY0(bbox.getBegin())
549  # `MaskedImage`s only support in-place addition, so rename for readability
550  residual += model[bbox]
551  newModel = residual
552  dcrModels.clampModel(subfilter, newModel, bbox, statsCtrl, self.config.regularizeSigma,
553  self.config.modelClampFactor, self.config.convergenceMaskPlanes)
554  dcrModels.conditionDcrModel(subfilter, newModel, bbox, gain=1.)
555  newModelImages.append(newModel)
556  return DcrModel(newModelImages, self.filterInfo)
557 
558  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
559  weightList, spanSetMaskList, statsCtrl):
560  """Calculate a quality of fit metric for the matched templates.
561 
562  Parameters
563  ----------
564  dcrModels : `lsst.pipe.tasks.DcrModel`
565  Best fit model of the true sky after correcting chromatic effects.
566  bbox : `lsst.afw.geom.box.Box2I`
567  Sub-region to coadd
568  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
569  The data references to the input warped exposures.
570  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
571  The image scalars correct for the zero point of the exposures.
572  weightList : `list` of `float`
573  The weight to give each input exposure in the coadd
574  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
575  Each element is dict with keys = mask plane name to add the spans to
576  statsCtrl : `lsst.afw.math.StatisticsControl`
577  Statistics control object for coadd
578 
579  Returns
580  -------
581  convergenceMetric : `float`
582  Quality of fit metric for all input exposures, within the sub-region
583  """
584  significanceImage = dcrModels.getReferenceImage(bbox)
585  tempExpName = self.getTempExpDatasetName(self.warpType)
586  weight = 0
587  metric = 0.
588  metricList = {}
589  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
590  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
591  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
592  imageScaler.scaleMaskedImage(exposure.maskedImage)
593  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
594  altMaskSpans=altMaskSpans)
595  metric += singleMetric*expWeight
596  metricList[tempExpRef.dataId["visit"]] = singleMetric
597  weight += expWeight
598  self.log.info("Individual metrics:\n%s", metricList)
599  return 1.0 if weight == 0.0 else metric/weight
600 
601  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
602  statsCtrl, altMaskSpans=None):
603  """Calculate a quality of fit metric for a single matched template.
604 
605  Parameters
606  ----------
607  dcrModels : `lsst.pipe.tasks.DcrModel`
608  Best fit model of the true sky after correcting chromatic effects.
609  exposure : `lsst.afw.image.ExposureF`
610  The input warped exposure to evaluate.
611  significanceImage : `numpy.ndarray`
612  Array of weights for each pixel corresponding to its significance
613  for the convergence calculation.
614  statsCtrl : `lsst.afw.math.StatisticsControl`
615  Statistics control object for coadd
616  altMaskSpans : `dict` containing spanSet lists, or None
617  The keys of the `dict` equal the mask plane name to add the spans to
618 
619  Returns
620  -------
621  convergenceMetric : `float`
622  Quality of fit metric for one exposure, within the sub-region.
623  """
624  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
625  templateImage = dcrModels.buildMatchedTemplate(self.warpCtrl,
626  visitInfo=exposure.getInfo().getVisitInfo(),
627  bbox=exposure.getBBox(),
628  wcs=exposure.getInfo().getWcs())
629  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
630  refVals = np.abs(templateImage.image.array)*significanceImage
631 
632  finitePixels = np.isfinite(diffVals)
633  if altMaskSpans is not None:
634  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
635  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
636  convergeMaskPixels = exposure.mask.array & convergeMask > 0
637  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
638  if np.sum(usePixels) == 0:
639  metric = 0.
640  else:
641  diffUse = diffVals[usePixels]
642  refUse = refVals[usePixels]
643  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
644  return metric
645 
646  def stackCoadd(self, dcrCoadds):
647  """Add a list of sub-band coadds together.
648 
649  Parameters
650  ----------
651  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
652  A list of coadd exposures, each exposure containing
653  the model for one subfilter.
654 
655  Returns
656  -------
657  coaddExposure : `lsst.afw.image.ExposureF`
658  A single coadd exposure that is the sum of the sub-bands.
659  """
660  coaddExposure = dcrCoadds[0].clone()
661  for coadd in dcrCoadds[1:]:
662  coaddExposure.maskedImage += coadd.maskedImage
663  return coaddExposure
664 
665  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None):
666  """Create a list of coadd exposures from a list of masked images.
667 
668  Parameters
669  ----------
670  dcrModels : `lsst.pipe.tasks.DcrModel`
671  Best fit model of the true sky after correcting chromatic effects.
672  skyInfo : `lsst.pipe.base.Struct`
673  Patch geometry information, from getSkyInfo
674  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
675  The data references to the input warped exposures.
676  weightList : `list` of `float`
677  The weight to give each input exposure in the coadd
678  calibration : `lsst.afw.Image.Calib`, optional
679  Scale factor to set the photometric zero point of an exposure.
680  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
681  A record of the observations that are included in the coadd.
682 
683  Returns
684  -------
685  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
686  A list of coadd exposures, each exposure containing
687  the model for one subfilter.
688  """
689  dcrCoadds = []
690  for model in dcrModels:
691  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
692  if calibration is not None:
693  coaddExposure.setCalib(calibration)
694  if coaddInputs is not None:
695  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
696  # Set the metadata for the coadd, including PSF and aperture corrections.
697  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
698  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
699  coaddExposure.setMaskedImage(model[skyInfo.bbox])
700  dcrCoadds.append(coaddExposure)
701  return dcrCoadds
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def dcrResiduals(self, residual, visitInfo, bbox, wcs)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def assemble(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def applyDcr(maskedImage, dcr, warpCtrl, bbox=None, useInverse=False)
Definition: dcrModel.py:425
def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateDcr(visitInfo, wcs, filterInfo, dcrNumSubfilters)
Definition: dcrModel.py:457
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList)