lsst.pipe.tasks  18.1.0-12-g18c6abff+2
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.meas.algorithms as measAlg
32 from lsst.meas.base import SingleFrameMeasurementTask
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf import MeasurePsfTask
37 
38 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
39 
40 
42  dcrNumSubfilters = pexConfig.Field(
43  dtype=int,
44  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
45  default=3,
46  )
47  maxNumIter = pexConfig.Field(
48  dtype=int,
49  optional=True,
50  doc="Maximum number of iterations of forward modeling.",
51  default=None,
52  )
53  minNumIter = pexConfig.Field(
54  dtype=int,
55  optional=True,
56  doc="Minimum number of iterations of forward modeling.",
57  default=None,
58  )
59  convergenceThreshold = pexConfig.Field(
60  dtype=float,
61  doc="Target relative change in convergence between iterations of forward modeling.",
62  default=0.001,
63  )
64  useConvergence = pexConfig.Field(
65  dtype=bool,
66  doc="Use convergence test as a forward modeling end condition?"
67  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
68  default=True,
69  )
70  baseGain = pexConfig.Field(
71  dtype=float,
72  optional=True,
73  doc="Relative weight to give the new solution vs. the last solution when updating the model."
74  "A value of 1.0 gives equal weight to both solutions."
75  "Small values imply slower convergence of the solution, but can "
76  "help prevent overshooting and failures in the fit."
77  "If ``baseGain`` is None, a conservative gain "
78  "will be calculated from the number of subfilters. ",
79  default=None,
80  )
81  useProgressiveGain = pexConfig.Field(
82  dtype=bool,
83  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
84  "When calculating the next gain, we use up to 5 previous gains and convergence values."
85  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
86  default=True,
87  )
88  doAirmassWeight = pexConfig.Field(
89  dtype=bool,
90  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
91  default=False,
92  )
93  modelWeightsWidth = pexConfig.Field(
94  dtype=float,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=3,
97  )
98  useModelWeights = pexConfig.Field(
99  dtype=bool,
100  doc="Width of the region around detected sources to include in the DcrModel.",
101  default=True,
102  )
103  splitSubfilters = pexConfig.Field(
104  dtype=bool,
105  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
106  "Instead of at the midpoint",
107  default=True,
108  )
109  splitThreshold = pexConfig.Field(
110  dtype=float,
111  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
112  "Set to 0 to always split the subfilters.",
113  default=0.1,
114  )
115  regularizeModelIterations = pexConfig.Field(
116  dtype=float,
117  doc="Maximum relative change of the model allowed between iterations."
118  "Set to zero to disable.",
119  default=2.,
120  )
121  regularizeModelFrequency = pexConfig.Field(
122  dtype=float,
123  doc="Maximum relative change of the model allowed between subfilters."
124  "Set to zero to disable.",
125  default=4.,
126  )
127  convergenceMaskPlanes = pexConfig.ListField(
128  dtype=str,
129  default=["DETECTED"],
130  doc="Mask planes to use to calculate convergence."
131  )
132  regularizationWidth = pexConfig.Field(
133  dtype=int,
134  default=2,
135  doc="Minimum radius of a region to include in regularization, in pixels."
136  )
137  imageInterpOrder = pexConfig.Field(
138  dtype=int,
139  doc="The order of the spline interpolation used to shift the image plane.",
140  default=3,
141  )
142  accelerateModel = pexConfig.Field(
143  dtype=float,
144  doc="Factor to amplify the differences between model planes by to speed convergence.",
145  default=3,
146  )
147  doCalculatePsf = pexConfig.Field(
148  dtype=bool,
149  doc="Set to detect stars and recalculate the PSF from the final coadd."
150  "Otherwise the PSF is estimated from a selection of the best input exposures",
151  default=True,
152  )
153  detectPsfSources = pexConfig.ConfigurableField(
154  target=measAlg.SourceDetectionTask,
155  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
156  )
157  measurePsfSources = pexConfig.ConfigurableField(
158  target=SingleFrameMeasurementTask,
159  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
160  )
161  measurePsf = pexConfig.ConfigurableField(
162  target=MeasurePsfTask,
163  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
164  )
165 
166  def setDefaults(self):
167  CompareWarpAssembleCoaddConfig.setDefaults(self)
168  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
169  self.doNImage = True
170  self.warpType = "direct"
171  self.assembleStaticSkyModel.warpType = self.warpType
172  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
173  self.assembleStaticSkyModel.doNImage = False
174  self.assembleStaticSkyModel.doWrite = False
175  self.detectPsfSources.returnOriginalFootprints = False
176  self.detectPsfSources.thresholdPolarity = "positive"
177  # Only use bright sources for PSF measurement
178  self.detectPsfSources.thresholdValue = 50
179  self.detectPsfSources.nSigmaToGrow = 2
180  # A valid star for PSF measurement should at least fill 5x5 pixels
181  self.detectPsfSources.minPixels = 25
182  # Use the variance plane to calculate signal to noise
183  self.detectPsfSources.thresholdType = "pixel_stdev"
184  # The signal to noise limit is good enough, while the flux limit is set
185  # in dimensionless units and may not be appropriate for all data sets.
186  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
187 
188 
190  """Assemble DCR coadded images from a set of warps.
191 
192  Attributes
193  ----------
194  bufferSize : `int`
195  The number of pixels to grow each subregion by to allow for DCR.
196 
197  Notes
198  -----
199  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
200  Warps (also called coadded temporary exposures), including the effects of
201  Differential Chromatic Refraction (DCR).
202  For full details of the mathematics and algorithm, please see
203  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
204 
205  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
206  each subfilter used in the iterative calculation.
207  It begins by dividing the bandpass-defining filter into N equal bandwidth
208  "subfilters", and divides the flux in each pixel from an initial coadd
209  equally into each as a "dcrModel". Because the airmass and parallactic
210  angle of each individual exposure is known, we can calculate the shift
211  relative to the center of the band in each subfilter due to DCR. For each
212  exposure we apply this shift as a linear transformation to the dcrModels
213  and stack the results to produce a DCR-matched exposure. The matched
214  exposures are subtracted from the input exposures to produce a set of
215  residual images, and these residuals are reverse shifted for each
216  exposures' subfilters and stacked. The shifted and stacked residuals are
217  added to the dcrModels to produce a new estimate of the flux in each pixel
218  within each subfilter. The dcrModels are solved for iteratively, which
219  continues until the solution from a new iteration improves by less than
220  a set percentage, or a maximum number of iterations is reached.
221  Two forms of regularization are employed to reduce unphysical results.
222  First, the new solution is averaged with the solution from the previous
223  iteration, which mitigates oscillating solutions where the model
224  overshoots with alternating very high and low values.
225  Second, a common degeneracy when the data have a limited range of airmass or
226  parallactic angle values is for one subfilter to be fit with very low or
227  negative values, while another subfilter is fit with very high values. This
228  typically appears in the form of holes next to sources in one subfilter,
229  and corresponding extended wings in another. Because each subfilter has
230  a narrow bandwidth we assume that physical sources that are above the noise
231  level will not vary in flux by more than a factor of `frequencyClampFactor`
232  between subfilters, and pixels that have flux deviations larger than that
233  factor will have the excess flux distributed evenly among all subfilters.
234  If `splitSubfilters` is set, then each subfilter will be further sub-
235  divided during the forward modeling step (only). This approximates using
236  a higher number of subfilters that may be necessary for high airmass
237  observations, but does not increase the number of free parameters in the
238  fit. This is needed when there are high airmass observations which would
239  otherwise have significant DCR even within a subfilter. Because calculating
240  the shifted images takes most of the time, splitting the subfilters is
241  turned off by way of the `splitThreshold` option for low-airmass
242  observations that do not suffer from DCR within a subfilter.
243  """
244 
245  ConfigClass = DcrAssembleCoaddConfig
246  _DefaultName = "dcrAssembleCoadd"
247 
248  def __init__(self, *args, **kwargs):
249  super().__init__(*args, **kwargs)
250  if self.config.doCalculatePsf:
251  self.schema = afwTable.SourceTable.makeMinimalSchema()
252  self.makeSubtask("detectPsfSources", schema=self.schema)
253  self.makeSubtask("measurePsfSources", schema=self.schema)
254  self.makeSubtask("measurePsf", schema=self.schema)
255 
256  @pipeBase.timeMethod
257  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258  """Assemble a coadd from a set of warps.
259 
260  Coadd a set of Warps. Compute weights to be applied to each Warp and
261  find scalings to match the photometric zeropoint to a reference Warp.
262  Assemble the Warps using run method.
263  Forward model chromatic effects across multiple subfilters,
264  and subtract from the input Warps to build sets of residuals.
265  Use the residuals to construct a new ``DcrModel`` for each subfilter,
266  and iterate until the model converges.
267  Interpolate over NaNs and optionally write the coadd to disk.
268  Return the coadded exposure.
269 
270  Parameters
271  ----------
272  dataRef : `lsst.daf.persistence.ButlerDataRef`
273  Data reference defining the patch for coaddition and the
274  reference Warp
275  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
276  List of data references to warps. Data to be coadded will be
277  selected from this list based on overlap with the patch defined by
278  the data reference.
279 
280  Returns
281  -------
282  results : `lsst.pipe.base.Struct`
283  The Struct contains the following fields:
284 
285  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
286  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
287  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
288  - ``dcrNImages``: `list` of exposure count images for each subfilter
289  """
290  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
291  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
292 
293  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294  warpRefList=warpRefList)
295  if results is None:
296  skyInfo = self.getSkyInfo(dataRef)
297  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
298  skyInfo.patchInfo.getIndex())
299  return
300  for subfilter in range(self.config.dcrNumSubfilters):
301  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
302  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
303  AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
304  if self.config.doWrite:
305  self.log.info("Persisting dcrCoadd")
306  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
307  numSubfilters=self.config.dcrNumSubfilters)
308  if self.config.doNImage and results.dcrNImages is not None:
309  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
310  numSubfilters=self.config.dcrNumSubfilters)
311 
312  return results
313 
314  def processResults(self, coaddExposure, dataRef):
315  """Interpolate over missing data and mask bright stars.
316 
317  Also detect sources on the coadd exposure and measure the final PSF,
318  if ``doCalculatePsf`` is set.
319 
320  Parameters
321  ----------
322  coaddExposure : `lsst.afw.image.Exposure`
323  The final coadded exposure.
324  dataRef : `lsst.daf.persistence.ButlerDataRef`
325  Data reference defining the patch for coaddition and the
326  reference Warp
327  """
328  super().processResults(coaddExposure, dataRef)
329 
330  if self.config.doCalculatePsf:
331  expId = dataRef.get("dcrCoaddId")
332  table = afwTable.SourceTable.make(self.schema)
333  detResults = self.detectPsfSources.run(table, coaddExposure, expId, clearMask=False)
334  coaddSources = detResults.sources
335  self.measurePsfSources.run(
336  measCat=coaddSources,
337  exposure=coaddExposure,
338  exposureId=expId
339  )
340  # Measure the PSF on the stacked subfilter coadds if possible.
341  # We should already have a decent estimate of the coadd PSF, however,
342  # so in case of any errors simply log them as a warning and use the
343  # default PSF.
344  try:
345  psfResults = self.measurePsf.run(coaddExposure, coaddSources, expId=expId)
346  except Exception as e:
347  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
348  else:
349  coaddExposure.setPsf(psfResults.psf)
350 
351  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
352  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
353 
354  Sets the property ``bufferSize``.
355 
356  Parameters
357  ----------
358  templateCoadd : `lsst.afw.image.ExposureF`
359  The initial coadd exposure before accounting for DCR.
360  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
361  The data references to the input warped exposures.
362  weightList : `list` of `float`
363  The weight to give each input exposure in the coadd
364  Will be modified in place if ``doAirmassWeight`` is set.
365 
366  Returns
367  -------
368  dcrModels : `lsst.pipe.tasks.DcrModel`
369  Best fit model of the true sky after correcting chromatic effects.
370 
371  Raises
372  ------
373  NotImplementedError
374  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
375  """
376  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
377  filterInfo = templateCoadd.getFilter()
378  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
379  raise NotImplementedError("No minimum/maximum wavelength information found"
380  " in the filter definition! Please add lambdaMin and lambdaMax"
381  " to the Mapper class in your obs package.")
382  tempExpName = self.getTempExpDatasetName(self.warpType)
383  dcrShifts = []
384  airmassDict = {}
385  angleDict = {}
386  psfSizeDict = {}
387  for visitNum, warpExpRef in enumerate(warpRefList):
388  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
389  visit = warpExpRef.dataId["visit"]
390  psf = warpExpRef.get(tempExpName).getPsf()
391  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
392  airmass = visitInfo.getBoresightAirmass()
393  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
394  airmassDict[visit] = airmass
395  angleDict[visit] = parallacticAngle
396  psfSizeDict[visit] = psfSize
397  if self.config.doAirmassWeight:
398  weightList[visitNum] *= airmass
399  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
400  filterInfo, self.config.dcrNumSubfilters))))
401  self.log.info("Selected airmasses:\n%s", airmassDict)
402  self.log.info("Selected parallactic angles:\n%s", angleDict)
403  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
404  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
405  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
406  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
407  self.config.dcrNumSubfilters,
408  filterInfo=filterInfo,
409  psf=psf)
410  return dcrModels
411 
412  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
413  supplementaryData=None):
414  """Assemble the coadd.
415 
416  Requires additional inputs Struct ``supplementaryData`` to contain a
417  ``templateCoadd`` that serves as the model of the static sky.
418 
419  Find artifacts and apply them to the warps' masks creating a list of
420  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
421  Then pass these alternative masks to the base class's assemble method.
422 
423  Divide the ``templateCoadd`` evenly between each subfilter of a
424  ``DcrModel`` as the starting best estimate of the true wavelength-
425  dependent sky. Forward model the ``DcrModel`` using the known
426  chromatic effects in each subfilter and calculate a convergence metric
427  based on how well the modeled template matches the input warps. If
428  the convergence has not yet reached the desired threshold, then shift
429  and stack the residual images to build a new ``DcrModel``. Apply
430  conditioning to prevent oscillating solutions between iterations or
431  between subfilters.
432 
433  Once the ``DcrModel`` reaches convergence or the maximum number of
434  iterations has been reached, fill the metadata for each subfilter
435  image and make them proper ``coaddExposure``s.
436 
437  Parameters
438  ----------
439  skyInfo : `lsst.pipe.base.Struct`
440  Patch geometry information, from getSkyInfo
441  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
442  The data references to the input warped exposures.
443  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
444  The image scalars correct for the zero point of the exposures.
445  weightList : `list` of `float`
446  The weight to give each input exposure in the coadd
447  supplementaryData : `lsst.pipe.base.Struct`
448  Result struct returned by ``makeSupplementaryData`` with components:
449 
450  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
451 
452  Returns
453  -------
454  result : `lsst.pipe.base.Struct`
455  Result struct with components:
456 
457  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
458  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
459  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
460  - ``dcrNImages``: `list` of exposure count images for each subfilter
461  """
462  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
463  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
464  templateCoadd = supplementaryData.templateCoadd
465  baseMask = templateCoadd.mask.clone()
466  # The variance plane is for each subfilter
467  # and should be proportionately lower than the full-band image
468  baseVariance = templateCoadd.variance.clone()
469  baseVariance /= self.config.dcrNumSubfilters
470  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
471  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
472  templateCoadd.setMask(baseMask)
473  badMaskPlanes = self.config.badMaskPlanes[:]
474  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
475  # This is because pixels in observations that are significantly affect by DCR
476  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
477  # but those are necessary to constrain the DCR model.
478  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
479 
480  stats = self.prepareStats(mask=badPixelMask)
481  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
482  if self.config.doNImage:
483  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
484  spanSetMaskList, stats.ctrl)
485  nImage = afwImage.ImageU(skyInfo.bbox)
486  # Note that this nImage will be a factor of dcrNumSubfilters higher than
487  # the nImage returned by assembleCoadd for most pixels. This is because each
488  # subfilter may have a different nImage, and fractional values are not allowed.
489  for dcrNImage in dcrNImages:
490  nImage += dcrNImage
491  else:
492  dcrNImages = None
493 
494  subregionSize = geom.Extent2I(*self.config.subregionSize)
495  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
496  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
497  subIter = 0
498  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
499  modelIter = 0
500  subIter += 1
501  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
502  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
503  dcrBBox = geom.Box2I(subBBox)
504  dcrBBox.grow(self.bufferSize)
505  dcrBBox.clip(dcrModels.bbox)
506  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
507  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
508  imageScalerList, spanSetMaskList)
509  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
510  warpRefList, weightList, stats.ctrl)
511  self.log.info("Initial convergence : %s", convergenceMetric)
512  convergenceList = [convergenceMetric]
513  gainList = []
514  convergenceCheck = 1.
515  refImage = templateCoadd.image
516  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
517  gain = self.calculateGain(convergenceList, gainList)
518  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
519  stats.ctrl, convergenceMetric, gain,
520  modelWeights, refImage, dcrWeights)
521  if self.config.useConvergence:
522  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
523  warpRefList, weightList, stats.ctrl)
524  if convergenceMetric == 0:
525  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
526  "most likely due to there being no valid data in the region.",
527  skyInfo.patchInfo.getIndex(), subIter)
528  break
529  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
530  if (convergenceCheck < 0) & (modelIter > minNumIter):
531  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
532  "iterations or desired convergence improvement of %s."
533  " Divergence: %s",
534  skyInfo.patchInfo.getIndex(), subIter,
535  self.config.convergenceThreshold, convergenceCheck)
536  break
537  convergenceList.append(convergenceMetric)
538  if modelIter > maxNumIter:
539  if self.config.useConvergence:
540  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
541  "before reaching desired convergence improvement of %s."
542  " Final convergence improvement: %s",
543  skyInfo.patchInfo.getIndex(), subIter,
544  self.config.convergenceThreshold, convergenceCheck)
545  break
546 
547  if self.config.useConvergence:
548  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
549  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
550  modelIter += 1
551  else:
552  if self.config.useConvergence:
553  self.log.info("Coadd patch %s subregion %s finished with "
554  "convergence metric %s after %s iterations",
555  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
556  else:
557  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
558  skyInfo.patchInfo.getIndex(), subIter, modelIter)
559  if self.config.useConvergence and convergenceMetric > 0:
560  self.log.info("Final convergence improvement was %.4f%% overall",
561  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
562 
563  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
564  calibration=self.scaleZeroPoint.getPhotoCalib(),
565  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
566  mask=baseMask,
567  variance=baseVariance)
568  coaddExposure = self.stackCoadd(dcrCoadds)
569  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
570  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
571 
572  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
573  """Calculate the number of exposures contributing to each subfilter.
574 
575  Parameters
576  ----------
577  dcrModels : `lsst.pipe.tasks.DcrModel`
578  Best fit model of the true sky after correcting chromatic effects.
579  bbox : `lsst.geom.box.Box2I`
580  Bounding box of the patch to coadd.
581  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
582  The data references to the input warped exposures.
583  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
584  Each element of the `dict` contains the new mask plane name
585  (e.g. "CLIPPED and/or "NO_DATA") as the key,
586  and the list of SpanSets to apply to the mask.
587  statsCtrl : `lsst.afw.math.StatisticsControl`
588  Statistics control object for coadd
589 
590  Returns
591  -------
592  dcrNImages : `list` of `lsst.afw.image.ImageU`
593  List of exposure count images for each subfilter
594  dcrWeights : `list` of `lsst.afw.image.ImageF`
595  Per-pixel weights for each subfilter.
596  Equal to 1/(number of unmasked images contributing to each pixel).
597  """
598  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
599  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
600  tempExpName = self.getTempExpDatasetName(self.warpType)
601  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
602  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
603  visitInfo = exposure.getInfo().getVisitInfo()
604  wcs = exposure.getInfo().getWcs()
605  mask = exposure.mask
606  if altMaskSpans is not None:
607  self.applyAltMaskPlanes(mask, altMaskSpans)
608  weightImage = np.zeros_like(exposure.image.array)
609  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
610  # The weights must be shifted in exactly the same way as the residuals,
611  # because they will be used as the denominator in the weighted average of residuals.
612  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
613  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
614  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
615  dcrWeight.array += shiftedWeights
616  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
617  weightsThreshold = 1.
618  goodPix = dcrWeights[0].array > weightsThreshold
619  for weights in dcrWeights[1:]:
620  goodPix = (weights.array > weightsThreshold) & goodPix
621  for subfilter in range(self.config.dcrNumSubfilters):
622  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
623  dcrWeights[subfilter].array[~goodPix] = 0.
624  dcrNImages[subfilter].array[~goodPix] = 0
625  return (dcrNImages, dcrWeights)
626 
627  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
628  statsCtrl, convergenceMetric,
629  gain, modelWeights, refImage, dcrWeights):
630  """Assemble the DCR coadd for a sub-region.
631 
632  Build a DCR-matched template for each input exposure, then shift the
633  residuals according to the DCR in each subfilter.
634  Stack the shifted residuals and apply them as a correction to the
635  solution from the previous iteration.
636  Restrict the new model solutions from varying by more than a factor of
637  `modelClampFactor` from the last solution, and additionally restrict the
638  individual subfilter models from varying by more than a factor of
639  `frequencyClampFactor` from their average.
640  Finally, mitigate potentially oscillating solutions by averaging the new
641  solution with the solution from the previous iteration, weighted by
642  their convergence metric.
643 
644  Parameters
645  ----------
646  dcrModels : `lsst.pipe.tasks.DcrModel`
647  Best fit model of the true sky after correcting chromatic effects.
648  subExposures : `dict` of `lsst.afw.image.ExposureF`
649  The pre-loaded exposures for the current subregion.
650  bbox : `lsst.geom.box.Box2I`
651  Bounding box of the subregion to coadd.
652  dcrBBox : `lsst.geom.box.Box2I`
653  Sub-region of the coadd which includes a buffer to allow for DCR.
654  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
655  The data references to the input warped exposures.
656  statsCtrl : `lsst.afw.math.StatisticsControl`
657  Statistics control object for coadd
658  convergenceMetric : `float`
659  Quality of fit metric for the matched templates of the input images.
660  gain : `float`, optional
661  Relative weight to give the new solution when updating the model.
662  modelWeights : `numpy.ndarray` or `float`
663  A 2D array of weight values that tapers smoothly to zero away from detected sources.
664  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
665  refImage : `lsst.afw.image.Image`
666  A reference image used to supply the default pixel values.
667  dcrWeights : `list` of `lsst.afw.image.Image`
668  Per-pixel weights for each subfilter.
669  Equal to 1/(number of unmasked images contributing to each pixel).
670  """
671  residualGeneratorList = []
672 
673  for warpExpRef in warpRefList:
674  exposure = subExposures[warpExpRef.dataId["visit"]]
675  visitInfo = exposure.getInfo().getVisitInfo()
676  wcs = exposure.getInfo().getWcs()
677  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
678  order=self.config.imageInterpOrder,
679  splitSubfilters=self.config.splitSubfilters,
680  splitThreshold=self.config.splitThreshold,
681  amplifyModel=self.config.accelerateModel)
682  residual = exposure.image.array - templateImage.array
683  # Note that the variance plane here is used to store weights, not the actual variance
684  residual *= exposure.variance.array
685  # The residuals are stored as a list of generators.
686  # This allows the residual for a given subfilter and exposure to be created
687  # on the fly, instead of needing to store them all in memory.
688  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
689 
690  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
691  gain=gain,
692  modelWeights=modelWeights,
693  refImage=refImage,
694  dcrWeights=dcrWeights)
695  dcrModels.assign(dcrSubModelOut, bbox)
696 
697  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
698  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
699 
700  Parameters
701  ----------
702  residual : `numpy.ndarray`
703  The residual masked image for one exposure,
704  after subtracting the matched template
705  visitInfo : `lsst.afw.image.VisitInfo`
706  Metadata for the exposure.
707  wcs : `lsst.afw.geom.SkyWcs`
708  Coordinate system definition (wcs) for the exposure.
709  filterInfo : `lsst.afw.image.Filter`
710  The filter definition, set in the current instruments' obs package.
711  Required for any calculation of DCR, including making matched templates.
712 
713  Yields
714  ------
715  residualImage : `numpy.ndarray`
716  The residual image for the next subfilter, shifted for DCR.
717  """
718  # Pre-calculate the spline-filtered residual image, so that step can be
719  # skipped in the shift calculation in `applyDcr`.
720  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
721  # Note that `splitSubfilters` is always turned off in the reverse direction.
722  # This option introduces additional blurring if applied to the residuals.
723  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
724  splitSubfilters=False)
725  for dcr in dcrShift:
726  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
727  doPrefilter=False, order=self.config.imageInterpOrder)
728 
729  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
730  gain, modelWeights, refImage, dcrWeights):
731  """Calculate a new DcrModel from a set of image residuals.
732 
733  Parameters
734  ----------
735  dcrModels : `lsst.pipe.tasks.DcrModel`
736  Current model of the true sky after correcting chromatic effects.
737  residualGeneratorList : `generator` of `numpy.ndarray`
738  The residual image for the next subfilter, shifted for DCR.
739  dcrBBox : `lsst.geom.box.Box2I`
740  Sub-region of the coadd which includes a buffer to allow for DCR.
741  statsCtrl : `lsst.afw.math.StatisticsControl`
742  Statistics control object for coadd
743  gain : `float`
744  Relative weight to give the new solution when updating the model.
745  modelWeights : `numpy.ndarray` or `float`
746  A 2D array of weight values that tapers smoothly to zero away from detected sources.
747  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
748  refImage : `lsst.afw.image.Image`
749  A reference image used to supply the default pixel values.
750  dcrWeights : `list` of `lsst.afw.image.Image`
751  Per-pixel weights for each subfilter.
752  Equal to 1/(number of unmasked images contributing to each pixel).
753 
754  Returns
755  -------
756  dcrModel : `lsst.pipe.tasks.DcrModel`
757  New model of the true sky after correcting chromatic effects.
758  """
759  newModelImages = []
760  for subfilter, model in enumerate(dcrModels):
761  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
762  residual = np.sum(residualsList, axis=0)
763  residual *= dcrWeights[subfilter][dcrBBox].array
764  # `MaskedImage`s only support in-place addition, so rename for readability
765  newModel = model[dcrBBox].clone()
766  newModel.array += residual
767  # Catch any invalid values
768  badPixels = ~np.isfinite(newModel.array)
769  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
770  if self.config.regularizeModelIterations > 0:
771  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
772  self.config.regularizeModelIterations,
773  self.config.regularizationWidth)
774  newModelImages.append(newModel)
775  if self.config.regularizeModelFrequency > 0:
776  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
777  self.config.regularizeModelFrequency,
778  self.config.regularizationWidth)
779  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
780  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
781  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
782  dcrModels.mask, dcrModels.variance)
783 
784  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
785  """Calculate a quality of fit metric for the matched templates.
786 
787  Parameters
788  ----------
789  dcrModels : `lsst.pipe.tasks.DcrModel`
790  Best fit model of the true sky after correcting chromatic effects.
791  subExposures : `dict` of `lsst.afw.image.ExposureF`
792  The pre-loaded exposures for the current subregion.
793  bbox : `lsst.geom.box.Box2I`
794  Sub-region to coadd
795  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
796  The data references to the input warped exposures.
797  weightList : `list` of `float`
798  The weight to give each input exposure in the coadd
799  statsCtrl : `lsst.afw.math.StatisticsControl`
800  Statistics control object for coadd
801 
802  Returns
803  -------
804  convergenceMetric : `float`
805  Quality of fit metric for all input exposures, within the sub-region
806  """
807  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
808  nSigma = 3.
809  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
810  bufferSize=self.bufferSize)
811  if np.max(significanceImage) == 0:
812  significanceImage += 1.
813  weight = 0
814  metric = 0.
815  metricList = {}
816  for warpExpRef, expWeight in zip(warpRefList, weightList):
817  exposure = subExposures[warpExpRef.dataId["visit"]][bbox]
818  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
819  metric += singleMetric
820  metricList[warpExpRef.dataId["visit"]] = singleMetric
821  weight += 1.
822  self.log.info("Individual metrics:\n%s", metricList)
823  return 1.0 if weight == 0.0 else metric/weight
824 
825  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
826  """Calculate a quality of fit metric for a single matched template.
827 
828  Parameters
829  ----------
830  dcrModels : `lsst.pipe.tasks.DcrModel`
831  Best fit model of the true sky after correcting chromatic effects.
832  exposure : `lsst.afw.image.ExposureF`
833  The input warped exposure to evaluate.
834  significanceImage : `numpy.ndarray`
835  Array of weights for each pixel corresponding to its significance
836  for the convergence calculation.
837  statsCtrl : `lsst.afw.math.StatisticsControl`
838  Statistics control object for coadd
839 
840  Returns
841  -------
842  convergenceMetric : `float`
843  Quality of fit metric for one exposure, within the sub-region.
844  """
845  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
846  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
847  order=self.config.imageInterpOrder,
848  splitSubfilters=self.config.splitSubfilters,
849  splitThreshold=self.config.splitThreshold,
850  amplifyModel=self.config.accelerateModel)
851  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
852  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
853 
854  finitePixels = np.isfinite(diffVals)
855  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
856  convergeMaskPixels = exposure.mask.array & convergeMask > 0
857  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
858  if np.sum(usePixels) == 0:
859  metric = 0.
860  else:
861  diffUse = diffVals[usePixels]
862  refUse = refVals[usePixels]
863  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
864  return metric
865 
866  def stackCoadd(self, dcrCoadds):
867  """Add a list of sub-band coadds together.
868 
869  Parameters
870  ----------
871  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
872  A list of coadd exposures, each exposure containing
873  the model for one subfilter.
874 
875  Returns
876  -------
877  coaddExposure : `lsst.afw.image.ExposureF`
878  A single coadd exposure that is the sum of the sub-bands.
879  """
880  coaddExposure = dcrCoadds[0].clone()
881  for coadd in dcrCoadds[1:]:
882  coaddExposure.maskedImage += coadd.maskedImage
883  return coaddExposure
884 
885  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
886  mask=None, variance=None):
887  """Create a list of coadd exposures from a list of masked images.
888 
889  Parameters
890  ----------
891  dcrModels : `lsst.pipe.tasks.DcrModel`
892  Best fit model of the true sky after correcting chromatic effects.
893  skyInfo : `lsst.pipe.base.Struct`
894  Patch geometry information, from getSkyInfo
895  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
896  The data references to the input warped exposures.
897  weightList : `list` of `float`
898  The weight to give each input exposure in the coadd
899  calibration : `lsst.afw.Image.PhotoCalib`, optional
900  Scale factor to set the photometric calibration of an exposure.
901  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
902  A record of the observations that are included in the coadd.
903  mask : `lsst.afw.image.Mask`, optional
904  Optional mask to override the values in the final coadd.
905  variance : `lsst.afw.image.Image`, optional
906  Optional variance plane to override the values in the final coadd.
907 
908  Returns
909  -------
910  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
911  A list of coadd exposures, each exposure containing
912  the model for one subfilter.
913  """
914  dcrCoadds = []
915  refModel = dcrModels.getReferenceImage()
916  for model in dcrModels:
917  if self.config.accelerateModel > 1:
918  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
919  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
920  if calibration is not None:
921  coaddExposure.setPhotoCalib(calibration)
922  if coaddInputs is not None:
923  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
924  # Set the metadata for the coadd, including PSF and aperture corrections.
925  self.assembleMetadata(coaddExposure, warpRefList, weightList)
926  # Overwrite the PSF
927  coaddExposure.setPsf(dcrModels.psf)
928  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
929  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
930  maskedImage.image = model
931  maskedImage.mask = dcrModels.mask
932  maskedImage.variance = dcrModels.variance
933  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
934  if mask is not None:
935  coaddExposure.setMask(mask)
936  if variance is not None:
937  coaddExposure.setVariance(variance)
938  dcrCoadds.append(coaddExposure)
939  return dcrCoadds
940 
941  def calculateGain(self, convergenceList, gainList):
942  """Calculate the gain to use for the current iteration.
943 
944  After calculating a new DcrModel, each value is averaged with the
945  value in the corresponding pixel from the previous iteration. This
946  reduces oscillating solutions that iterative techniques are plagued by,
947  and speeds convergence. By far the biggest changes to the model
948  happen in the first couple iterations, so we can also use a more
949  aggressive gain later when the model is changing slowly.
950 
951  Parameters
952  ----------
953  convergenceList : `list` of `float`
954  The quality of fit metric from each previous iteration.
955  gainList : `list` of `float`
956  The gains used in each previous iteration: appended with the new
957  gain value.
958  Gains are numbers between ``self.config.baseGain`` and 1.
959 
960  Returns
961  -------
962  gain : `float`
963  Relative weight to give the new solution when updating the model.
964  A value of 1.0 gives equal weight to both solutions.
965 
966  Raises
967  ------
968  ValueError
969  If ``len(convergenceList) != len(gainList)+1``.
970  """
971  nIter = len(convergenceList)
972  if nIter != len(gainList) + 1:
973  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
974  % (len(convergenceList), len(gainList)))
975 
976  if self.config.baseGain is None:
977  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
978  # The more subfilters being modeled, the lower the gain should be.
979  baseGain = 1./(self.config.dcrNumSubfilters - 1)
980  else:
981  baseGain = self.config.baseGain
982 
983  if self.config.useProgressiveGain and nIter > 2:
984  # To calculate the best gain to use, compare the past gains that have been used
985  # with the resulting convergences to estimate the best gain to use.
986  # Algorithmically, this is a Kalman filter.
987  # If forward modeling proceeds perfectly, the convergence metric should
988  # asymptotically approach a final value.
989  # We can estimate that value from the measured changes in convergence
990  # weighted by the gains used in each previous iteration.
991  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
992  for i in range(nIter - 1)]
993  # The convergence metric is strictly positive, so if the estimated final convergence is
994  # less than zero, force it to zero.
995  estFinalConv = np.array(estFinalConv)
996  estFinalConv[estFinalConv < 0] = 0
997  # Because the estimate may slowly change over time, only use the most recent measurements.
998  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
999  lastGain = gainList[-1]
1000  lastConv = convergenceList[-2]
1001  newConv = convergenceList[-1]
1002  # The predicted convergence is the value we would get if the new model calculated
1003  # in the previous iteration was perfect. Recall that the updated model that is
1004  # actually used is the gain-weighted average of the new and old model,
1005  # so the convergence would be similarly weighted.
1006  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1007  # If the measured and predicted convergence are very close, that indicates
1008  # that our forward model is accurate and we can use a more aggressive gain
1009  # If the measured convergence is significantly worse (or better!) than predicted,
1010  # that indicates that the model is not converging as expected and
1011  # we should use a more conservative gain.
1012  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1013  newGain = 1 - abs(delta)
1014  # Average the gains to prevent oscillating solutions.
1015  newGain = (newGain + lastGain)/2.
1016  gain = max(baseGain, newGain)
1017  else:
1018  gain = baseGain
1019  gainList.append(gain)
1020  return gain
1021 
1022  def calculateModelWeights(self, dcrModels, dcrBBox):
1023  """Build an array that smoothly tapers to 0 away from detected sources.
1024 
1025  Parameters
1026  ----------
1027  dcrModels : `lsst.pipe.tasks.DcrModel`
1028  Best fit model of the true sky after correcting chromatic effects.
1029  dcrBBox : `lsst.geom.box.Box2I`
1030  Sub-region of the coadd which includes a buffer to allow for DCR.
1031 
1032  Returns
1033  -------
1034  weights : `numpy.ndarray` or `float`
1035  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1036  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1037 
1038  Raises
1039  ------
1040  ValueError
1041  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1042  """
1043  if not self.config.useModelWeights:
1044  return 1.0
1045  if self.config.modelWeightsWidth < 0:
1046  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1047  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1048  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1049  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1050  weights[convergeMaskPixels] = 1.
1051  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1052  weights /= np.max(weights)
1053  return weights
1054 
1055  def applyModelWeights(self, modelImages, refImage, modelWeights):
1056  """Smoothly replace model pixel values with those from a
1057  reference at locations away from detected sources.
1058 
1059  Parameters
1060  ----------
1061  modelImages : `list` of `lsst.afw.image.Image`
1062  The new DCR model images from the current iteration.
1063  The values will be modified in place.
1064  refImage : `lsst.afw.image.MaskedImage`
1065  A reference image used to supply the default pixel values.
1066  modelWeights : `numpy.ndarray` or `float`
1067  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1068  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1069  """
1070  if self.config.useModelWeights:
1071  for model in modelImages:
1072  model.array *= modelWeights
1073  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1074 
1075  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1076  """Pre-load sub-regions of a list of exposures.
1077 
1078  Parameters
1079  ----------
1080  bbox : `lsst.geom.box.Box2I`
1081  Sub-region to coadd
1082  statsCtrl : `lsst.afw.math.StatisticsControl`
1083  Statistics control object for coadd
1084  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1085  The data references to the input warped exposures.
1086  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1087  The image scalars correct for the zero point of the exposures.
1088  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1089  Each element is dict with keys = mask plane name to add the spans to
1090 
1091  Returns
1092  -------
1093  subExposures : `dict`
1094  The `dict` keys are the visit IDs,
1095  and the values are `lsst.afw.image.ExposureF`
1096  The pre-loaded exposures for the current subregion.
1097  The variance plane contains weights, and not the variance
1098  """
1099  tempExpName = self.getTempExpDatasetName(self.warpType)
1100  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1101  subExposures = {}
1102  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1103  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1104  if altMaskSpans is not None:
1105  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1106  imageScaler.scaleMaskedImage(exposure.maskedImage)
1107  # Note that the variance plane here is used to store weights, not the actual variance
1108  exposure.variance.array[:, :] = 0.
1109  # Set the weight of unmasked pixels to 1.
1110  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1111  # Set the image value of masked pixels to zero.
1112  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1113  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1114  subExposures[warpExpRef.dataId["visit"]] = exposure
1115  return subExposures
1116 
1117  def selectCoaddPsf(self, templateCoadd, warpRefList):
1118  """Compute the PSF of the coadd from the exposures with the best seeing.
1119 
1120  Parameters
1121  ----------
1122  templateCoadd : `lsst.afw.image.ExposureF`
1123  The initial coadd exposure before accounting for DCR.
1124  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1125  The data references to the input warped exposures.
1126 
1127  Returns
1128  -------
1129  psf : `lsst.meas.algorithms.CoaddPsf`
1130  The average PSF of the input exposures with the best seeing.
1131  """
1132  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1133  tempExpName = self.getTempExpDatasetName(self.warpType)
1134  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1135  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1136  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1137  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1138  psfSizes = np.zeros(len(ccds))
1139  ccdVisits = np.array(ccds["visit"])
1140  for warpExpRef in warpRefList:
1141  psf = warpExpRef.get(tempExpName).getPsf()
1142  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1143  visit = warpExpRef.dataId["visit"]
1144  psfSizes[ccdVisits == visit] = psfSize
1145  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1146  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1147  # of the mean or median FWHM of the input exposures.
1148  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1149  goodPsfs = psfSizes <= sizeThreshold
1150  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1151  self.config.coaddPsf.makeControl())
1152  return psf
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)