lsst.pipe.tasks  16.0-13-g1e751bcc+3
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 import numpy as np
24 import lsst.afw.geom as afwGeom
25 import lsst.afw.image as afwImage
26 import lsst.afw.math as afwMath
27 import lsst.coadd.utils as coaddUtils
28 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
29 import lsst.pex.config as pexConfig
30 import lsst.pipe.base as pipeBase
31 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
32 
33 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
34 
35 
37  dcrNumSubfilters = pexConfig.Field(
38  dtype=int,
39  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
40  default=3,
41  )
42  maxNumIter = pexConfig.Field(
43  dtype=int,
44  doc="Maximum number of iterations of forward modeling.",
45  default=8,
46  )
47  minNumIter = pexConfig.Field(
48  dtype=int,
49  doc="Minimum number of iterations of forward modeling.",
50  default=3,
51  )
52  convergenceThreshold = pexConfig.Field(
53  dtype=float,
54  doc="Target relative change in convergence between iterations of forward modeling.",
55  default=0.001,
56  )
57  useConvergence = pexConfig.Field(
58  dtype=bool,
59  doc="Use convergence test as a forward modeling end condition?"
60  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
61  default=True,
62  )
63  doAirmassWeight = pexConfig.Field(
64  dtype=bool,
65  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
66  default=True,
67  )
68  modelClampFactor = pexConfig.Field(
69  dtype=float,
70  doc="Maximum relative change of the model allowed between iterations.",
71  default=2.,
72  )
73  regularizeSigma = pexConfig.Field(
74  dtype=float,
75  doc="Threshold to exclude noise-like pixels from regularization.",
76  default=3.,
77  )
78  clampFrequency = pexConfig.Field(
79  dtype=float,
80  doc="Maximum relative change of the model allowed between subfilters.",
81  default=2.,
82  )
83  maxGain = pexConfig.Field(
84  dtype=float,
85  doc="Maximum convergence-weighted gain to apply between forward modeling iterations.",
86  default=2.,
87  )
88  minGain = pexConfig.Field(
89  dtype=float,
90  doc="Minimum convergence-weighted gain to apply between forward modeling iterations.",
91  default=0.5,
92  )
93  convergenceMaskPlanes = pexConfig.ListField(
94  dtype=str,
95  default=["DETECTED"],
96  doc="Mask planes to use to calculate convergence."
97  )
98  imageWarpMethod = pexConfig.Field(
99  dtype=str,
100  doc="Name of the warping kernel to use for shifting the image and variance planes.",
101  default="lanczos3",
102  )
103  maskWarpMethod = pexConfig.Field(
104  dtype=str,
105  doc="Name of the warping kernel to use for shifting the mask plane.",
106  default="bilinear",
107  )
108 
109  def setDefaults(self):
110  CompareWarpAssembleCoaddConfig.setDefaults(self)
111  self.doNImage = True
112  self.warpType = 'direct'
113  self.assembleStaticSkyModel.warpType = self.warpType
114  self.assembleStaticSkyModel.doNImage = self.doNImage
115  self.statistic = 'MEAN'
116 
117 
119  """Assemble DCR coadded images from a set of warps.
120 
121  Notes
122  -----
123  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
124  Warps (also called coadded temporary exposures), including the effects of
125  Differential Chromatic Refraction (DCR).
126  For full details of the mathematics and algorithm, please see
127  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
128 
129  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
130  each subfilter used in the iterative calculation.
131  It begins by dividing the bandpass-defining filter into N equal bandwidth
132  "subfilters", and divides the flux in each pixel from an initial coadd
133  equally into each as a "dcrModel". Because the airmass and parallactic
134  angle of each individual exposure is known, we can calculate the shift
135  relative to the center of the band in each subfilter due to DCR. For each
136  exposure we apply this shift as a linear transformation to the dcrModels
137  and stack the results to produce a DCR-matched exposure. The matched
138  exposures are subtracted from the input exposures to produce a set of
139  residual images, and these residuals are reverse shifted for each
140  exposures' subfilters and stacked. The shifted and stacked residuals are
141  added to the dcrModels to produce a new estimate of the flux in each pixel
142  within each subfilter. The dcrModels are solved for iteratively, which
143  continues until the solution from a new iteration improves by less than
144  a set percentage, or a maximum number of iterations is reached.
145  Two forms of regularization are employed to reduce unphysical results.
146  First, the new solution is averaged with the solution from the previous
147  iteration, which mitigates oscillating solutions where the model
148  overshoots with alternating very high and low values.
149  Second, a common degeneracy when the data have a limited range of airmass or
150  parallactic angle values is for one subfilter to be fit with very low or
151  negative values, while another subfilter is fit with very high values. This
152  typically appears in the form of holes next to sources in one subfilter,
153  and corresponding extended wings in another. Because each subfilter has
154  a narrow bandwidth we assume that physical sources that are above the noise
155  level will not vary in flux by more than a factor of `clampFrequency`
156  between subfilters, and pixels that have flux deviations larger than that
157  factor will have the excess flux distributed evenly among all subfilters.
158  """
159 
160  ConfigClass = DcrAssembleCoaddConfig
161  _DefaultName = "dcrAssembleCoadd"
162 
163  @pipeBase.timeMethod
164  def runDataRef(self, dataRef, selectDataList=[]):
165  """Assemble a coadd from a set of warps.
166 
167  Coadd a set of Warps. Compute weights to be applied to each Warp and
168  find scalings to match the photometric zeropoint to a reference Warp.
169  Assemble the Warps using run method.
170  Forward model chromatic effects across multiple subfilters,
171  and subtract from the input Warps to build sets of residuals.
172  Use the residuals to construct a new ``DcrModel`` for each subfilter,
173  and iterate until the model converges.
174  Interpolate over NaNs and optionally write the coadd to disk.
175  Return the coadded exposure.
176 
177  Parameters
178  ----------
179  dataRef : `lsst.daf.persistence.ButlerDataRef`
180  Data reference defining the patch for coaddition and the
181  reference Warp
182  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
183  List of data references to warps. Data to be coadded will be
184  selected from this list based on overlap with the patch defined by
185  the data reference.
186 
187  Returns
188  -------
189  results : `lsst.pipe.base.Struct`
190  The Struct contains the following fields:
191 
192  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
193  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
194  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
195  - ``dcrNImages``: `list` of exposure count images for each subfilter
196  """
197  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
198  for subfilter in range(self.config.dcrNumSubfilters):
199  self.processResults(results.dcrCoadds[subfilter], dataRef)
200  if self.config.doWrite:
201  self.log.info("Persisting dcrCoadd")
202  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
203  numSubfilters=self.config.dcrNumSubfilters)
204  if self.config.doNImage and results.dcrNImages is not None:
205  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
206  numSubfilters=self.config.dcrNumSubfilters)
207 
208  return results
209 
210  def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList):
211  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
212 
213  Sets the properties ``filterInfo`` and ``bufferSize``.
214 
215  Parameters
216  ----------
217  templateCoadd : `lsst.afw.image.ExposureF`
218  The initial coadd exposure before accounting for DCR.
219  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
220  The data references to the input warped exposures.
221  weightList : `list` of `float`
222  The weight to give each input exposure in the coadd
223  Will be modified in place if ``doAirmassWeight`` is set.
224 
225  Returns
226  -------
227  dcrModels : `lsst.pipe.tasks.DcrModel`
228  Best fit model of the true sky after correcting chromatic effects.
229 
230  Raises
231  ------
232  NotImplementedError
233  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
234  """
235  filterInfo = templateCoadd.getFilter()
236  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
237  raise NotImplementedError("No minimum/maximum wavelength information found"
238  " in the filter definition! Please add lambdaMin and lambdaMax"
239  " to the Mapper class in your obs package.")
240  tempExpName = self.getTempExpDatasetName(self.warpType)
241  dcrShifts = []
242  for visitNum, tempExpRef in enumerate(tempExpRefList):
243  visitInfo = tempExpRef.get(tempExpName + "_visitInfo")
244  airmass = visitInfo.getBoresightAirmass()
245  if self.config.doAirmassWeight:
246  weightList[visitNum] *= airmass
247  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
248  filterInfo, self.config.dcrNumSubfilters))))
249  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
250  # Turn off the warping cache, since we set the linear interpolation length to the entire subregion
251  # This warper is only used for applying DCR shifts, which are assumed to be uniform across a patch
252  warpCache = 0
253  warpInterpLength = max(self.config.subregionSize)
254  self.warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
255  self.config.maskWarpMethod,
256  cacheSize=warpCache, interpLength=warpInterpLength)
257  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
258  self.config.dcrNumSubfilters,
259  filterInfo=filterInfo,
260  psf=templateCoadd.getPsf())
261  return dcrModels
262 
263  def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
264  supplementaryData=None):
265  """Assemble the coadd.
266 
267  Requires additional inputs Struct ``supplementaryData`` to contain a
268  ``templateCoadd`` that serves as the model of the static sky.
269 
270  Find artifacts and apply them to the warps' masks creating a list of
271  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
272  Then pass these alternative masks to the base class's assemble method.
273 
274  Divide the ``templateCoadd`` evenly between each subfilter of a
275  ``DcrModel`` as the starting best estimate of the true wavelength-
276  dependent sky. Forward model the ``DcrModel`` using the known
277  chromatic effects in each subfilter and calculate a convergence metric
278  based on how well the modeled template matches the input warps. If
279  the convergence has not yet reached the desired threshold, then shift
280  and stack the residual images to build a new ``DcrModel``. Apply
281  conditioning to prevent oscillating solutions between iterations or
282  between subfilters.
283 
284  Once the ``DcrModel`` reaches convergence or the maximum number of
285  iterations has been reached, fill the metadata for each subfilter
286  image and make them proper ``coaddExposure``s.
287 
288  Parameters
289  ----------
290  skyInfo : `lsst.pipe.base.Struct`
291  Patch geometry information, from getSkyInfo
292  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
293  The data references to the input warped exposures.
294  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
295  The image scalars correct for the zero point of the exposures.
296  weightList : `list` of `float`
297  The weight to give each input exposure in the coadd
298  supplementaryData : `lsst.pipe.base.Struct`
299  Result struct returned by ``makeSupplementaryData`` with components:
300 
301  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
302 
303  Returns
304  -------
305  result : `lsst.pipe.base.Struct`
306  Result struct with components:
307 
308  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
309  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
310  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
311  - ``dcrNImages``: `list` of exposure count images for each subfilter
312  """
313  templateCoadd = supplementaryData.templateCoadd
314  spanSetMaskList = self.findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
315  badMaskPlanes = self.config.badMaskPlanes[:]
316  badMaskPlanes.append("CLIPPED")
317  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
318  # Propagate PSF-matched EDGE pixels to coadd SENSOR_EDGE and INEXACT_PSF
319  # Psf-Matching moves the real edge inwards
320  self.applyAltEdgeMask(templateCoadd.mask, spanSetMaskList)
321 
322  stats = self.prepareStats(mask=badPixelMask)
323  dcrModels = self.prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
324  if self.config.doNImage:
325  dcrNImages = self.calculateNImage(dcrModels, skyInfo.bbox,
326  tempExpRefList, spanSetMaskList, stats.ctrl)
327  nImage = afwImage.ImageU(skyInfo.bbox)
328  # Note that this nImage will be a factor of dcrNumSubfilters higher than
329  # the nImage returned by assembleCoadd for most pixels. This is because each
330  # subfilter may have a different nImage, and fractional values are not allowed.
331  for dcrNImage in dcrNImages:
332  nImage += dcrNImage
333  else:
334  dcrNImages = None
335 
336  baseMask = templateCoadd.mask
337  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
338  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
339  modelIter = 0
340  self.log.info("Computing coadd over %s", subBBox)
341  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
342  imageScalerList, weightList, spanSetMaskList,
343  stats.ctrl)
344  self.log.info("Initial convergence : %s", convergenceMetric)
345  convergenceList = [convergenceMetric]
346  convergenceCheck = 1.
347  subfilterVariance = None
348  while (convergenceCheck > self.config.convergenceThreshold or
349  modelIter < self.config.minNumIter):
350  self.dcrAssembleSubregion(dcrModels, subBBox, tempExpRefList, imageScalerList,
351  weightList, spanSetMaskList, stats.flags, stats.ctrl,
352  convergenceMetric, baseMask, subfilterVariance)
353  if self.config.useConvergence:
354  convergenceMetric = self.calculateConvergence(dcrModels, subBBox, tempExpRefList,
355  imageScalerList, weightList,
356  spanSetMaskList,
357  stats.ctrl)
358  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
359  convergenceList.append(convergenceMetric)
360  if modelIter > self.config.maxNumIter:
361  if self.config.useConvergence:
362  self.log.warn("Coadd %s reached maximum iterations before reaching"
363  " desired convergence improvement of %s."
364  " Final convergence improvement: %s",
365  subBBox, self.config.convergenceThreshold, convergenceCheck)
366  break
367 
368  if self.config.useConvergence:
369  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement",
370  modelIter, convergenceMetric, 100.*convergenceCheck)
371  modelIter += 1
372  else:
373  if self.config.useConvergence:
374  self.log.info("Coadd %s finished with convergence metric %s after %s iterations",
375  subBBox, convergenceMetric, modelIter)
376  else:
377  self.log.info("Coadd %s finished after %s iterations", subBBox, modelIter)
378  if self.config.useConvergence:
379  self.log.info("Final convergence improvement was %.4f%% overall",
380  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
381  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
382  calibration=self.scaleZeroPoint.getCalib(),
383  coaddInputs=self.inputRecorder.makeCoaddInputs())
384  coaddExposure = self.stackCoadd(dcrCoadds)
385  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
386  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
387 
388  def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
389  """Calculate the number of exposures contributing to each subfilter.
390 
391  Parameters
392  ----------
393  dcrModels : `lsst.pipe.tasks.DcrModel`
394  Best fit model of the true sky after correcting chromatic effects.
395  bbox : `lsst.afw.geom.box.Box2I`
396  Bounding box of the patch to coadd.
397  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
398  The data references to the input warped exposures.
399  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
400  Each element is dict with keys = mask plane name to add the spans to
401  statsCtrl : `lsst.afw.math.StatisticsControl`
402  Statistics control object for coadd
403 
404  Returns
405  -------
406  dcrNImages : `list` of `lsst.afw.image.ImageU`
407  List of exposure count images for each subfilter
408  """
409  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
410  tempExpName = self.getTempExpDatasetName(self.warpType)
411  for tempExpRef, altMaskSpans in zip(tempExpRefList, spanSetMaskList):
412  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
413  visitInfo = exposure.getInfo().getVisitInfo()
414  wcs = exposure.getInfo().getWcs()
415  mask = exposure.mask
416  if altMaskSpans is not None:
417  self.applyAltMaskPlanes(mask, altMaskSpans)
418  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
419  for dcr, dcrNImage in zip(dcrShift, dcrNImages):
420  shiftedImage = applyDcr(exposure.maskedImage, dcr, self.warpCtrl, useInverse=True)
421  dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
422  return dcrNImages
423 
424  def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList,
425  spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
426  baseMask, subfilterVariance):
427  """Assemble the DCR coadd for a sub-region.
428 
429  Build a DCR-matched template for each input exposure, then shift the
430  residuals according to the DCR in each subfilter.
431  Stack the shifted residuals and apply them as a correction to the
432  solution from the previous iteration.
433  Restrict the new model solutions from varying by more than a factor of
434  `modelClampFactor` from the last solution, and additionally restrict the
435  individual subfilter models from varying by more than a factor of
436  `clampFrequency` from their average.
437  Finally, mitigate potentially oscillating solutions by averaging the new
438  solution with the solution from the previous iteration, weighted by
439  their convergence metric.
440 
441  Parameters
442  ----------
443  dcrModels : `lsst.pipe.tasks.DcrModel`
444  Best fit model of the true sky after correcting chromatic effects.
445  bbox : `lsst.afw.geom.box.Box2I`
446  Bounding box of the subregion to coadd.
447  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
448  The data references to the input warped exposures.
449  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
450  The image scalars correct for the zero point of the exposures.
451  weightList : `list` of `float`
452  The weight to give each input exposure in the coadd
453  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
454  Each element is dict with keys = mask plane name to add the spans to
455  statsFlags : `lsst.afw.math.Property`
456  Statistics settings for coaddition.
457  statsCtrl : `lsst.afw.math.StatisticsControl`
458  Statistics control object for coadd
459  convergenceMetric : `float`
460  Quality of fit metric for the matched templates of the input images.
461  baseMask : `lsst.afw.image.Mask`
462  Mask of the initial template coadd.
463  subfilterVariance : `list` of `numpy.ndarray`
464  The variance of each coadded subfilter image.
465  """
466  bboxGrow = afwGeom.Box2I(bbox)
467  bboxGrow.grow(self.bufferSize)
468  bboxGrow.clip(dcrModels.bbox)
469 
470  tempExpName = self.getTempExpDatasetName(self.warpType)
471  residualGeneratorList = []
472 
473  for tempExpRef, imageScaler, altMaskSpans in zip(tempExpRefList, imageScalerList, spanSetMaskList):
474  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bboxGrow)
475  visitInfo = exposure.getInfo().getVisitInfo()
476  wcs = exposure.getInfo().getWcs()
477  maskedImage = exposure.maskedImage
478  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl, visitInfo=visitInfo,
479  bbox=bboxGrow, wcs=wcs, mask=baseMask)
480  imageScaler.scaleMaskedImage(maskedImage)
481  if altMaskSpans is not None:
482  self.applyAltMaskPlanes(maskedImage.mask, altMaskSpans)
483 
484  if self.config.removeMaskPlanes:
485  self.removeMaskPlanes(maskedImage)
486  maskedImage -= templateImage
487  residualGeneratorList.append(self.dcrResiduals(maskedImage, visitInfo, bboxGrow, wcs,
488  dcrModels.filter))
489 
490  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, bboxGrow,
491  statsFlags, statsCtrl, weightList)
492  dcrSubModelOut.regularizeModel(bboxGrow, baseMask, statsCtrl, self.config.regularizeSigma,
493  self.config.clampFrequency, self.config.convergenceMaskPlanes)
494  dcrModels.assign(dcrSubModelOut, bbox)
495 
496  def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo):
497  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
498 
499  Parameters
500  ----------
501  residual : `lsst.afw.image.MaskedImageF`
502  The residual masked image for one exposure,
503  after subtracting the matched template
504  visitInfo : `lsst.afw.image.VisitInfo`
505  Metadata for the exposure.
506  bbox : `lsst.afw.geom.box.Box2I`
507  Sub-region of the coadd
508  wcs : `lsst.afw.geom.SkyWcs`
509  Coordinate system definition (wcs) for the exposure.
510  filterInfo : `lsst.afw.image.Filter`
511  The filter definition, set in the current instruments' obs package.
512  Required for any calculation of DCR, including making matched templates.
513 
514  Yields
515  ------
516  residualImage : `lsst.afw.image.maskedImageF`
517  The residual image for the next subfilter, shifted for DCR.
518  """
519  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
520  for dcr in dcrShift:
521  yield applyDcr(residual, dcr, self.warpCtrl, bbox=bbox, useInverse=True)
522 
523  def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList):
524  """Calculate a new DcrModel from a set of image residuals.
525 
526  Parameters
527  ----------
528  dcrModels : `lsst.pipe.tasks.DcrModel`
529  Current model of the true sky after correcting chromatic effects.
530  residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF`
531  The residual image for the next subfilter, shifted for DCR.
532  bbox : `lsst.afw.geom.box.Box2I`
533  Sub-region of the coadd
534  statsFlags : `lsst.afw.math.Property`
535  Statistics settings for coaddition.
536  statsCtrl : `lsst.afw.math.StatisticsControl`
537  Statistics control object for coadd
538  weightList : `list` of `float`
539  The weight to give each input exposure in the coadd
540 
541  Returns
542  -------
543  dcrModel : `lsst.pipe.tasks.DcrModel`
544  New model of the true sky after correcting chromatic effects.
545  """
546  maskMap = self.setRejectedMaskMapping(statsCtrl)
547  clipped = dcrModels[0].mask.getPlaneBitMask("CLIPPED")
548  newModelImages = []
549  for subfilter, model in enumerate(dcrModels):
550  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
551  residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
552  clipped, maskMap)
553  residual.setXY0(bbox.getBegin())
554  # `MaskedImage`s only support in-place addition, so rename for readability
555  residual += model[bbox]
556  newModel = residual
557  dcrModels.clampModel(subfilter, newModel, bbox, statsCtrl, self.config.regularizeSigma,
558  self.config.modelClampFactor, self.config.convergenceMaskPlanes)
559  dcrModels.conditionDcrModel(subfilter, newModel, bbox, gain=1.)
560  newModelImages.append(newModel)
561  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
562 
563  def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList,
564  weightList, spanSetMaskList, statsCtrl):
565  """Calculate a quality of fit metric for the matched templates.
566 
567  Parameters
568  ----------
569  dcrModels : `lsst.pipe.tasks.DcrModel`
570  Best fit model of the true sky after correcting chromatic effects.
571  bbox : `lsst.afw.geom.box.Box2I`
572  Sub-region to coadd
573  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
574  The data references to the input warped exposures.
575  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
576  The image scalars correct for the zero point of the exposures.
577  weightList : `list` of `float`
578  The weight to give each input exposure in the coadd
579  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
580  Each element is dict with keys = mask plane name to add the spans to
581  statsCtrl : `lsst.afw.math.StatisticsControl`
582  Statistics control object for coadd
583 
584  Returns
585  -------
586  convergenceMetric : `float`
587  Quality of fit metric for all input exposures, within the sub-region
588  """
589  significanceImage = dcrModels.getReferenceImage(bbox)
590  tempExpName = self.getTempExpDatasetName(self.warpType)
591  weight = 0
592  metric = 0.
593  metricList = {}
594  zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
595  for tempExpRef, expWeight, imageScaler, altMaskSpans in zipIterables:
596  exposure = tempExpRef.get(tempExpName + "_sub", bbox=bbox)
597  imageScaler.scaleMaskedImage(exposure.maskedImage)
598  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl,
599  altMaskSpans=altMaskSpans)
600  metric += singleMetric*expWeight
601  metricList[tempExpRef.dataId["visit"]] = singleMetric
602  weight += expWeight
603  self.log.info("Individual metrics:\n%s", metricList)
604  return 1.0 if weight == 0.0 else metric/weight
605 
606  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage,
607  statsCtrl, altMaskSpans=None):
608  """Calculate a quality of fit metric for a single matched template.
609 
610  Parameters
611  ----------
612  dcrModels : `lsst.pipe.tasks.DcrModel`
613  Best fit model of the true sky after correcting chromatic effects.
614  exposure : `lsst.afw.image.ExposureF`
615  The input warped exposure to evaluate.
616  significanceImage : `numpy.ndarray`
617  Array of weights for each pixel corresponding to its significance
618  for the convergence calculation.
619  statsCtrl : `lsst.afw.math.StatisticsControl`
620  Statistics control object for coadd
621  altMaskSpans : `dict` containing spanSet lists, or None
622  The keys of the `dict` equal the mask plane name to add the spans to
623 
624  Returns
625  -------
626  convergenceMetric : `float`
627  Quality of fit metric for one exposure, within the sub-region.
628  """
629  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
630  templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.warpCtrl,
631  visitInfo=exposure.getInfo().getVisitInfo(),
632  bbox=exposure.getBBox(),
633  wcs=exposure.getInfo().getWcs())
634  diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
635  refVals = np.abs(templateImage.image.array)*significanceImage
636 
637  finitePixels = np.isfinite(diffVals)
638  if altMaskSpans is not None:
639  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
640  goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
641  convergeMaskPixels = exposure.mask.array & convergeMask > 0
642  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
643  if np.sum(usePixels) == 0:
644  metric = 0.
645  else:
646  diffUse = diffVals[usePixels]
647  refUse = refVals[usePixels]
648  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
649  return metric
650 
651  def stackCoadd(self, dcrCoadds):
652  """Add a list of sub-band coadds together.
653 
654  Parameters
655  ----------
656  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
657  A list of coadd exposures, each exposure containing
658  the model for one subfilter.
659 
660  Returns
661  -------
662  coaddExposure : `lsst.afw.image.ExposureF`
663  A single coadd exposure that is the sum of the sub-bands.
664  """
665  coaddExposure = dcrCoadds[0].clone()
666  for coadd in dcrCoadds[1:]:
667  coaddExposure.maskedImage += coadd.maskedImage
668  return coaddExposure
669 
670  def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None):
671  """Create a list of coadd exposures from a list of masked images.
672 
673  Parameters
674  ----------
675  dcrModels : `lsst.pipe.tasks.DcrModel`
676  Best fit model of the true sky after correcting chromatic effects.
677  skyInfo : `lsst.pipe.base.Struct`
678  Patch geometry information, from getSkyInfo
679  tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
680  The data references to the input warped exposures.
681  weightList : `list` of `float`
682  The weight to give each input exposure in the coadd
683  calibration : `lsst.afw.Image.Calib`, optional
684  Scale factor to set the photometric zero point of an exposure.
685  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
686  A record of the observations that are included in the coadd.
687 
688  Returns
689  -------
690  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
691  A list of coadd exposures, each exposure containing
692  the model for one subfilter.
693  """
694  dcrCoadds = []
695  for model in dcrModels:
696  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
697  if calibration is not None:
698  coaddExposure.setCalib(calibration)
699  if coaddInputs is not None:
700  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
701  # Set the metadata for the coadd, including PSF and aperture corrections.
702  self.assembleMetadata(coaddExposure, tempExpRefList, weightList)
703  coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
704  coaddExposure.setMaskedImage(model[skyInfo.bbox])
705  dcrCoadds.append(coaddExposure)
706  return dcrCoadds
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:186
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def dcrAssembleSubregion(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList)