lsst.pipe.tasks  18.0.0-8-g24ce6f0f+5
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.meas.algorithms as measAlg
32 from lsst.meas.base import SingleFrameMeasurementTask
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf import MeasurePsfTask
37 
38 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
39 
40 
42  dcrNumSubfilters = pexConfig.Field(
43  dtype=int,
44  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
45  default=3,
46  )
47  maxNumIter = pexConfig.Field(
48  dtype=int,
49  optional=True,
50  doc="Maximum number of iterations of forward modeling.",
51  default=None,
52  )
53  minNumIter = pexConfig.Field(
54  dtype=int,
55  optional=True,
56  doc="Minimum number of iterations of forward modeling.",
57  default=None,
58  )
59  convergenceThreshold = pexConfig.Field(
60  dtype=float,
61  doc="Target relative change in convergence between iterations of forward modeling.",
62  default=0.001,
63  )
64  useConvergence = pexConfig.Field(
65  dtype=bool,
66  doc="Use convergence test as a forward modeling end condition?"
67  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
68  default=True,
69  )
70  baseGain = pexConfig.Field(
71  dtype=float,
72  optional=True,
73  doc="Relative weight to give the new solution vs. the last solution when updating the model."
74  "A value of 1.0 gives equal weight to both solutions."
75  "Small values imply slower convergence of the solution, but can "
76  "help prevent overshooting and failures in the fit."
77  "If ``baseGain`` is None, a conservative gain "
78  "will be calculated from the number of subfilters. ",
79  default=None,
80  )
81  useProgressiveGain = pexConfig.Field(
82  dtype=bool,
83  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
84  "When calculating the next gain, we use up to 5 previous gains and convergence values."
85  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
86  default=True,
87  )
88  doAirmassWeight = pexConfig.Field(
89  dtype=bool,
90  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
91  default=False,
92  )
93  modelWeightsWidth = pexConfig.Field(
94  dtype=float,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=3,
97  )
98  useModelWeights = pexConfig.Field(
99  dtype=bool,
100  doc="Width of the region around detected sources to include in the DcrModel.",
101  default=True,
102  )
103  splitSubfilters = pexConfig.Field(
104  dtype=bool,
105  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
106  "Instead of at the midpoint",
107  default=True,
108  )
109  splitThreshold = pexConfig.Field(
110  dtype=float,
111  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
112  "Set to 0 to always split the subfilters.",
113  default=0.1,
114  )
115  regularizeModelIterations = pexConfig.Field(
116  dtype=float,
117  doc="Maximum relative change of the model allowed between iterations."
118  "Set to zero to disable.",
119  default=2.,
120  )
121  regularizeModelFrequency = pexConfig.Field(
122  dtype=float,
123  doc="Maximum relative change of the model allowed between subfilters."
124  "Set to zero to disable.",
125  default=4.,
126  )
127  convergenceMaskPlanes = pexConfig.ListField(
128  dtype=str,
129  default=["DETECTED"],
130  doc="Mask planes to use to calculate convergence."
131  )
132  regularizationWidth = pexConfig.Field(
133  dtype=int,
134  default=2,
135  doc="Minimum radius of a region to include in regularization, in pixels."
136  )
137  imageInterpOrder = pexConfig.Field(
138  dtype=int,
139  doc="The order of the spline interpolation used to shift the image plane.",
140  default=3,
141  )
142  accelerateModel = pexConfig.Field(
143  dtype=float,
144  doc="Factor to amplify the differences between model planes by to speed convergence.",
145  default=3,
146  )
147  doCalculatePsf = pexConfig.Field(
148  dtype=bool,
149  doc="Set to detect stars and recalculate the PSF from the final coadd."
150  "Otherwise the PSF is estimated from a selection of the best input exposures",
151  default=True,
152  )
153  detectPsfSources = pexConfig.ConfigurableField(
154  target=measAlg.SourceDetectionTask,
155  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
156  )
157  measurePsfSources = pexConfig.ConfigurableField(
158  target=SingleFrameMeasurementTask,
159  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
160  )
161  measurePsf = pexConfig.ConfigurableField(
162  target=MeasurePsfTask,
163  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
164  )
165 
166  def setDefaults(self):
167  CompareWarpAssembleCoaddConfig.setDefaults(self)
168  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
169  self.doNImage = True
170  self.warpType = "direct"
171  self.assembleStaticSkyModel.warpType = self.warpType
172  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
173  self.assembleStaticSkyModel.doNImage = False
174  self.assembleStaticSkyModel.doWrite = False
175  self.detectPsfSources.returnOriginalFootprints = False
176  self.detectPsfSources.thresholdPolarity = "positive"
177  # Only use bright sources for PSF measurement
178  self.detectPsfSources.thresholdValue = 50
179  self.detectPsfSources.nSigmaToGrow = 2
180  # A valid star for PSF measurement should at least fill 5x5 pixels
181  self.detectPsfSources.minPixels = 25
182  # Use the variance plane to calculate signal to noise
183  self.detectPsfSources.thresholdType = "pixel_stdev"
184  # The signal to noise limit is good enough, while the flux limit is set
185  # in dimensionless units and may not be appropriate for all data sets.
186  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
187 
188 
190  """Assemble DCR coadded images from a set of warps.
191 
192  Attributes
193  ----------
194  bufferSize : `int`
195  The number of pixels to grow each subregion by to allow for DCR.
196 
197  Notes
198  -----
199  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
200  Warps (also called coadded temporary exposures), including the effects of
201  Differential Chromatic Refraction (DCR).
202  For full details of the mathematics and algorithm, please see
203  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
204 
205  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
206  each subfilter used in the iterative calculation.
207  It begins by dividing the bandpass-defining filter into N equal bandwidth
208  "subfilters", and divides the flux in each pixel from an initial coadd
209  equally into each as a "dcrModel". Because the airmass and parallactic
210  angle of each individual exposure is known, we can calculate the shift
211  relative to the center of the band in each subfilter due to DCR. For each
212  exposure we apply this shift as a linear transformation to the dcrModels
213  and stack the results to produce a DCR-matched exposure. The matched
214  exposures are subtracted from the input exposures to produce a set of
215  residual images, and these residuals are reverse shifted for each
216  exposures' subfilters and stacked. The shifted and stacked residuals are
217  added to the dcrModels to produce a new estimate of the flux in each pixel
218  within each subfilter. The dcrModels are solved for iteratively, which
219  continues until the solution from a new iteration improves by less than
220  a set percentage, or a maximum number of iterations is reached.
221  Two forms of regularization are employed to reduce unphysical results.
222  First, the new solution is averaged with the solution from the previous
223  iteration, which mitigates oscillating solutions where the model
224  overshoots with alternating very high and low values.
225  Second, a common degeneracy when the data have a limited range of airmass or
226  parallactic angle values is for one subfilter to be fit with very low or
227  negative values, while another subfilter is fit with very high values. This
228  typically appears in the form of holes next to sources in one subfilter,
229  and corresponding extended wings in another. Because each subfilter has
230  a narrow bandwidth we assume that physical sources that are above the noise
231  level will not vary in flux by more than a factor of `frequencyClampFactor`
232  between subfilters, and pixels that have flux deviations larger than that
233  factor will have the excess flux distributed evenly among all subfilters.
234  If `splitSubfilters` is set, then each subfilter will be further sub-
235  divided during the forward modeling step (only). This approximates using
236  a higher number of subfilters that may be necessary for high airmass
237  observations, but does not increase the number of free parameters in the
238  fit. This is needed when there are high airmass observations which would
239  otherwise have significant DCR even within a subfilter. Because calculating
240  the shifted images takes most of the time, splitting the subfilters is
241  turned off by way of the `splitThreshold` option for low-airmass
242  observations that do not suffer from DCR within a subfilter.
243  """
244 
245  ConfigClass = DcrAssembleCoaddConfig
246  _DefaultName = "dcrAssembleCoadd"
247 
248  def __init__(self, *args, **kwargs):
249  super().__init__(*args, **kwargs)
250  if self.config.doCalculatePsf:
251  self.schema = afwTable.SourceTable.makeMinimalSchema()
252  self.makeSubtask("detectPsfSources", schema=self.schema)
253  self.makeSubtask("measurePsfSources", schema=self.schema)
254  self.makeSubtask("measurePsf", schema=self.schema)
255 
256  @pipeBase.timeMethod
257  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258  """Assemble a coadd from a set of warps.
259 
260  Coadd a set of Warps. Compute weights to be applied to each Warp and
261  find scalings to match the photometric zeropoint to a reference Warp.
262  Assemble the Warps using run method.
263  Forward model chromatic effects across multiple subfilters,
264  and subtract from the input Warps to build sets of residuals.
265  Use the residuals to construct a new ``DcrModel`` for each subfilter,
266  and iterate until the model converges.
267  Interpolate over NaNs and optionally write the coadd to disk.
268  Return the coadded exposure.
269 
270  Parameters
271  ----------
272  dataRef : `lsst.daf.persistence.ButlerDataRef`
273  Data reference defining the patch for coaddition and the
274  reference Warp
275  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
276  List of data references to warps. Data to be coadded will be
277  selected from this list based on overlap with the patch defined by
278  the data reference.
279 
280  Returns
281  -------
282  results : `lsst.pipe.base.Struct`
283  The Struct contains the following fields:
284 
285  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
286  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
287  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
288  - ``dcrNImages``: `list` of exposure count images for each subfilter
289  """
290  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
291  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
292 
293  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294  warpRefList=warpRefList)
295  if results is None:
296  skyInfo = self.getSkyInfo(dataRef)
297  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
298  skyInfo.patchInfo.getIndex())
299  return
300  for subfilter in range(self.config.dcrNumSubfilters):
301  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
302  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
303  AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
304  if self.config.doWrite:
305  self.log.info("Persisting dcrCoadd")
306  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
307  numSubfilters=self.config.dcrNumSubfilters)
308  if self.config.doNImage and results.dcrNImages is not None:
309  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
310  numSubfilters=self.config.dcrNumSubfilters)
311 
312  return results
313 
314  def processResults(self, coaddExposure, dataRef):
315  """Interpolate over missing data and mask bright stars.
316 
317  Also detect sources on the coadd exposure and measure the final PSF,
318  if ``doCalculatePsf`` is set.
319 
320  Parameters
321  ----------
322  coaddExposure : `lsst.afw.image.Exposure`
323  The final coadded exposure.
324  dataRef : `lsst.daf.persistence.ButlerDataRef`
325  Data reference defining the patch for coaddition and the
326  reference Warp
327  """
328  super().processResults(coaddExposure, dataRef)
329 
330  if self.config.doCalculatePsf:
331  expId = dataRef.get("dcrCoaddId")
332  table = afwTable.SourceTable.make(self.schema)
333  detResults = self.detectPsfSources.run(table, coaddExposure, expId, clearMask=False)
334  coaddSources = detResults.sources
335  self.measurePsfSources.run(
336  measCat=coaddSources,
337  exposure=coaddExposure,
338  exposureId=expId
339  )
340  try:
341  psfResults = self.measurePsf.run(coaddExposure, coaddSources, expId=expId)
342  except RuntimeError as e:
343  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
344  else:
345  coaddExposure.setPsf(psfResults.psf)
346 
347  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
348  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
349 
350  Sets the property ``bufferSize``.
351 
352  Parameters
353  ----------
354  templateCoadd : `lsst.afw.image.ExposureF`
355  The initial coadd exposure before accounting for DCR.
356  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
357  The data references to the input warped exposures.
358  weightList : `list` of `float`
359  The weight to give each input exposure in the coadd
360  Will be modified in place if ``doAirmassWeight`` is set.
361 
362  Returns
363  -------
364  dcrModels : `lsst.pipe.tasks.DcrModel`
365  Best fit model of the true sky after correcting chromatic effects.
366 
367  Raises
368  ------
369  NotImplementedError
370  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
371  """
372  filterInfo = templateCoadd.getFilter()
373  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
374  raise NotImplementedError("No minimum/maximum wavelength information found"
375  " in the filter definition! Please add lambdaMin and lambdaMax"
376  " to the Mapper class in your obs package.")
377  tempExpName = self.getTempExpDatasetName(self.warpType)
378  dcrShifts = []
379  airmassDict = {}
380  angleDict = {}
381  for visitNum, warpExpRef in enumerate(warpRefList):
382  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
383  visit = warpExpRef.dataId["visit"]
384  airmass = visitInfo.getBoresightAirmass()
385  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
386  airmassDict[visit] = airmass
387  angleDict[visit] = parallacticAngle
388  if self.config.doAirmassWeight:
389  weightList[visitNum] *= airmass
390  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
391  filterInfo, self.config.dcrNumSubfilters))))
392  self.log.info("Selected airmasses:\n%s", airmassDict)
393  self.log.info("Selected parallactic angles:\n%s", angleDict)
394  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
395  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
396  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
397  self.config.dcrNumSubfilters,
398  filterInfo=filterInfo,
399  psf=psf)
400  return dcrModels
401 
402  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
403  supplementaryData=None):
404  """Assemble the coadd.
405 
406  Requires additional inputs Struct ``supplementaryData`` to contain a
407  ``templateCoadd`` that serves as the model of the static sky.
408 
409  Find artifacts and apply them to the warps' masks creating a list of
410  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
411  Then pass these alternative masks to the base class's assemble method.
412 
413  Divide the ``templateCoadd`` evenly between each subfilter of a
414  ``DcrModel`` as the starting best estimate of the true wavelength-
415  dependent sky. Forward model the ``DcrModel`` using the known
416  chromatic effects in each subfilter and calculate a convergence metric
417  based on how well the modeled template matches the input warps. If
418  the convergence has not yet reached the desired threshold, then shift
419  and stack the residual images to build a new ``DcrModel``. Apply
420  conditioning to prevent oscillating solutions between iterations or
421  between subfilters.
422 
423  Once the ``DcrModel`` reaches convergence or the maximum number of
424  iterations has been reached, fill the metadata for each subfilter
425  image and make them proper ``coaddExposure``s.
426 
427  Parameters
428  ----------
429  skyInfo : `lsst.pipe.base.Struct`
430  Patch geometry information, from getSkyInfo
431  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
432  The data references to the input warped exposures.
433  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
434  The image scalars correct for the zero point of the exposures.
435  weightList : `list` of `float`
436  The weight to give each input exposure in the coadd
437  supplementaryData : `lsst.pipe.base.Struct`
438  Result struct returned by ``makeSupplementaryData`` with components:
439 
440  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
441 
442  Returns
443  -------
444  result : `lsst.pipe.base.Struct`
445  Result struct with components:
446 
447  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
448  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
449  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
450  - ``dcrNImages``: `list` of exposure count images for each subfilter
451  """
452  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
453  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
454  templateCoadd = supplementaryData.templateCoadd
455  baseMask = templateCoadd.mask.clone()
456  # The variance plane is for each subfilter
457  # and should be proportionately lower than the full-band image
458  baseVariance = templateCoadd.variance.clone()
459  baseVariance /= self.config.dcrNumSubfilters
460  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
461  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
462  templateCoadd.setMask(baseMask)
463  badMaskPlanes = self.config.badMaskPlanes[:]
464  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
465  # This is because pixels in observations that are significantly affect by DCR
466  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
467  # but those are necessary to constrain the DCR model.
468  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
469 
470  stats = self.prepareStats(mask=badPixelMask)
471  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
472  if self.config.doNImage:
473  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
474  spanSetMaskList, stats.ctrl)
475  nImage = afwImage.ImageU(skyInfo.bbox)
476  # Note that this nImage will be a factor of dcrNumSubfilters higher than
477  # the nImage returned by assembleCoadd for most pixels. This is because each
478  # subfilter may have a different nImage, and fractional values are not allowed.
479  for dcrNImage in dcrNImages:
480  nImage += dcrNImage
481  else:
482  dcrNImages = None
483 
484  subregionSize = geom.Extent2I(*self.config.subregionSize)
485  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
486  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
487  subIter = 0
488  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
489  modelIter = 0
490  subIter += 1
491  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
492  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
493  dcrBBox = geom.Box2I(subBBox)
494  dcrBBox.grow(self.bufferSize)
495  dcrBBox.clip(dcrModels.bbox)
496  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
497  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
498  imageScalerList, spanSetMaskList)
499  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
500  warpRefList, weightList, stats.ctrl)
501  self.log.info("Initial convergence : %s", convergenceMetric)
502  convergenceList = [convergenceMetric]
503  gainList = []
504  convergenceCheck = 1.
505  refImage = templateCoadd.image
506  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
507  gain = self.calculateGain(convergenceList, gainList)
508  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
509  stats.ctrl, convergenceMetric, gain,
510  modelWeights, refImage, dcrWeights)
511  if self.config.useConvergence:
512  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
513  warpRefList, weightList, stats.ctrl)
514  if convergenceMetric == 0:
515  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
516  "most likely due to there being no valid data in the region.",
517  skyInfo.patchInfo.getIndex(), subIter)
518  break
519  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
520  if (convergenceCheck < 0) & (modelIter > minNumIter):
521  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
522  "iterations or desired convergence improvement of %s."
523  " Divergence: %s",
524  skyInfo.patchInfo.getIndex(), subIter,
525  self.config.convergenceThreshold, convergenceCheck)
526  break
527  convergenceList.append(convergenceMetric)
528  if modelIter > maxNumIter:
529  if self.config.useConvergence:
530  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
531  "before reaching desired convergence improvement of %s."
532  " Final convergence improvement: %s",
533  skyInfo.patchInfo.getIndex(), subIter,
534  self.config.convergenceThreshold, convergenceCheck)
535  break
536 
537  if self.config.useConvergence:
538  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
539  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
540  modelIter += 1
541  else:
542  if self.config.useConvergence:
543  self.log.info("Coadd patch %s subregion %s finished with "
544  "convergence metric %s after %s iterations",
545  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
546  else:
547  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
548  skyInfo.patchInfo.getIndex(), subIter, modelIter)
549  if self.config.useConvergence and convergenceMetric > 0:
550  self.log.info("Final convergence improvement was %.4f%% overall",
551  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
552 
553  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
554  calibration=self.scaleZeroPoint.getPhotoCalib(),
555  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
556  mask=baseMask,
557  variance=baseVariance)
558  coaddExposure = self.stackCoadd(dcrCoadds)
559  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
560  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
561 
562  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
563  """Calculate the number of exposures contributing to each subfilter.
564 
565  Parameters
566  ----------
567  dcrModels : `lsst.pipe.tasks.DcrModel`
568  Best fit model of the true sky after correcting chromatic effects.
569  bbox : `lsst.geom.box.Box2I`
570  Bounding box of the patch to coadd.
571  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
572  The data references to the input warped exposures.
573  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
574  Each element of the `dict` contains the new mask plane name
575  (e.g. "CLIPPED and/or "NO_DATA") as the key,
576  and the list of SpanSets to apply to the mask.
577  statsCtrl : `lsst.afw.math.StatisticsControl`
578  Statistics control object for coadd
579 
580  Returns
581  -------
582  dcrNImages : `list` of `lsst.afw.image.ImageU`
583  List of exposure count images for each subfilter
584  dcrWeights : `list` of `lsst.afw.image.ImageF`
585  Per-pixel weights for each subfilter.
586  Equal to 1/(number of unmasked images contributing to each pixel).
587  """
588  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
589  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
590  tempExpName = self.getTempExpDatasetName(self.warpType)
591  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
592  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
593  visitInfo = exposure.getInfo().getVisitInfo()
594  wcs = exposure.getInfo().getWcs()
595  mask = exposure.mask
596  if altMaskSpans is not None:
597  self.applyAltMaskPlanes(mask, altMaskSpans)
598  weightImage = np.zeros_like(exposure.image.array)
599  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
600  # The weights must be shifted in exactly the same way as the residuals,
601  # because they will be used as the denominator in the weighted average of residuals.
602  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
603  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
604  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
605  dcrWeight.array += shiftedWeights
606  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
607  weightsThreshold = 1.
608  goodPix = dcrWeights[0].array > weightsThreshold
609  for weights in dcrWeights[1:]:
610  goodPix = (weights.array > weightsThreshold) & goodPix
611  for subfilter in range(self.config.dcrNumSubfilters):
612  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
613  dcrWeights[subfilter].array[~goodPix] = 0.
614  dcrNImages[subfilter].array[~goodPix] = 0
615  return (dcrNImages, dcrWeights)
616 
617  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
618  statsCtrl, convergenceMetric,
619  gain, modelWeights, refImage, dcrWeights):
620  """Assemble the DCR coadd for a sub-region.
621 
622  Build a DCR-matched template for each input exposure, then shift the
623  residuals according to the DCR in each subfilter.
624  Stack the shifted residuals and apply them as a correction to the
625  solution from the previous iteration.
626  Restrict the new model solutions from varying by more than a factor of
627  `modelClampFactor` from the last solution, and additionally restrict the
628  individual subfilter models from varying by more than a factor of
629  `frequencyClampFactor` from their average.
630  Finally, mitigate potentially oscillating solutions by averaging the new
631  solution with the solution from the previous iteration, weighted by
632  their convergence metric.
633 
634  Parameters
635  ----------
636  dcrModels : `lsst.pipe.tasks.DcrModel`
637  Best fit model of the true sky after correcting chromatic effects.
638  subExposures : `dict` of `lsst.afw.image.ExposureF`
639  The pre-loaded exposures for the current subregion.
640  bbox : `lsst.geom.box.Box2I`
641  Bounding box of the subregion to coadd.
642  dcrBBox : `lsst.geom.box.Box2I`
643  Sub-region of the coadd which includes a buffer to allow for DCR.
644  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
645  The data references to the input warped exposures.
646  statsCtrl : `lsst.afw.math.StatisticsControl`
647  Statistics control object for coadd
648  convergenceMetric : `float`
649  Quality of fit metric for the matched templates of the input images.
650  gain : `float`, optional
651  Relative weight to give the new solution when updating the model.
652  modelWeights : `numpy.ndarray` or `float`
653  A 2D array of weight values that tapers smoothly to zero away from detected sources.
654  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
655  refImage : `lsst.afw.image.Image`
656  A reference image used to supply the default pixel values.
657  dcrWeights : `list` of `lsst.afw.image.Image`
658  Per-pixel weights for each subfilter.
659  Equal to 1/(number of unmasked images contributing to each pixel).
660  """
661  residualGeneratorList = []
662 
663  for warpExpRef in warpRefList:
664  exposure = subExposures[warpExpRef.dataId["visit"]]
665  visitInfo = exposure.getInfo().getVisitInfo()
666  wcs = exposure.getInfo().getWcs()
667  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
668  order=self.config.imageInterpOrder,
669  splitSubfilters=self.config.splitSubfilters,
670  splitThreshold=self.config.splitThreshold,
671  amplifyModel=self.config.accelerateModel)
672  residual = exposure.image.array - templateImage.array
673  # Note that the variance plane here is used to store weights, not the actual variance
674  residual *= exposure.variance.array
675  # The residuals are stored as a list of generators.
676  # This allows the residual for a given subfilter and exposure to be created
677  # on the fly, instead of needing to store them all in memory.
678  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
679 
680  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
681  gain=gain,
682  modelWeights=modelWeights,
683  refImage=refImage,
684  dcrWeights=dcrWeights)
685  dcrModels.assign(dcrSubModelOut, bbox)
686 
687  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
688  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
689 
690  Parameters
691  ----------
692  residual : `numpy.ndarray`
693  The residual masked image for one exposure,
694  after subtracting the matched template
695  visitInfo : `lsst.afw.image.VisitInfo`
696  Metadata for the exposure.
697  wcs : `lsst.afw.geom.SkyWcs`
698  Coordinate system definition (wcs) for the exposure.
699  filterInfo : `lsst.afw.image.Filter`
700  The filter definition, set in the current instruments' obs package.
701  Required for any calculation of DCR, including making matched templates.
702 
703  Yields
704  ------
705  residualImage : `numpy.ndarray`
706  The residual image for the next subfilter, shifted for DCR.
707  """
708  # Pre-calculate the spline-filtered residual image, so that step can be
709  # skipped in the shift calculation in `applyDcr`.
710  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
711  # Note that `splitSubfilters` is always turned off in the reverse direction.
712  # This option introduces additional blurring if applied to the residuals.
713  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
714  splitSubfilters=False)
715  for dcr in dcrShift:
716  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
717  doPrefilter=False, order=self.config.imageInterpOrder)
718 
719  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
720  gain, modelWeights, refImage, dcrWeights):
721  """Calculate a new DcrModel from a set of image residuals.
722 
723  Parameters
724  ----------
725  dcrModels : `lsst.pipe.tasks.DcrModel`
726  Current model of the true sky after correcting chromatic effects.
727  residualGeneratorList : `generator` of `numpy.ndarray`
728  The residual image for the next subfilter, shifted for DCR.
729  dcrBBox : `lsst.geom.box.Box2I`
730  Sub-region of the coadd which includes a buffer to allow for DCR.
731  statsCtrl : `lsst.afw.math.StatisticsControl`
732  Statistics control object for coadd
733  gain : `float`
734  Relative weight to give the new solution when updating the model.
735  modelWeights : `numpy.ndarray` or `float`
736  A 2D array of weight values that tapers smoothly to zero away from detected sources.
737  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
738  refImage : `lsst.afw.image.Image`
739  A reference image used to supply the default pixel values.
740  dcrWeights : `list` of `lsst.afw.image.Image`
741  Per-pixel weights for each subfilter.
742  Equal to 1/(number of unmasked images contributing to each pixel).
743 
744  Returns
745  -------
746  dcrModel : `lsst.pipe.tasks.DcrModel`
747  New model of the true sky after correcting chromatic effects.
748  """
749  newModelImages = []
750  for subfilter, model in enumerate(dcrModels):
751  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
752  residual = np.sum(residualsList, axis=0)
753  residual *= dcrWeights[subfilter][dcrBBox].array
754  # `MaskedImage`s only support in-place addition, so rename for readability
755  newModel = model[dcrBBox].clone()
756  newModel.array += residual
757  # Catch any invalid values
758  badPixels = ~np.isfinite(newModel.array)
759  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
760  if self.config.regularizeModelIterations > 0:
761  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
762  self.config.regularizeModelIterations,
763  self.config.regularizationWidth)
764  newModelImages.append(newModel)
765  if self.config.regularizeModelFrequency > 0:
766  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
767  self.config.regularizeModelFrequency,
768  self.config.regularizationWidth)
769  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
770  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
771  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
772  dcrModels.mask, dcrModels.variance)
773 
774  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
775  """Calculate a quality of fit metric for the matched templates.
776 
777  Parameters
778  ----------
779  dcrModels : `lsst.pipe.tasks.DcrModel`
780  Best fit model of the true sky after correcting chromatic effects.
781  subExposures : `dict` of `lsst.afw.image.ExposureF`
782  The pre-loaded exposures for the current subregion.
783  bbox : `lsst.geom.box.Box2I`
784  Sub-region to coadd
785  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
786  The data references to the input warped exposures.
787  weightList : `list` of `float`
788  The weight to give each input exposure in the coadd
789  statsCtrl : `lsst.afw.math.StatisticsControl`
790  Statistics control object for coadd
791 
792  Returns
793  -------
794  convergenceMetric : `float`
795  Quality of fit metric for all input exposures, within the sub-region
796  """
797  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
798  nSigma = 3.
799  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
800  bufferSize=self.bufferSize)
801  if np.max(significanceImage) == 0:
802  significanceImage += 1.
803  weight = 0
804  metric = 0.
805  metricList = {}
806  for warpExpRef, expWeight in zip(warpRefList, weightList):
807  exposure = subExposures[warpExpRef.dataId["visit"]][bbox]
808  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
809  metric += singleMetric
810  metricList[warpExpRef.dataId["visit"]] = singleMetric
811  weight += 1.
812  self.log.info("Individual metrics:\n%s", metricList)
813  return 1.0 if weight == 0.0 else metric/weight
814 
815  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
816  """Calculate a quality of fit metric for a single matched template.
817 
818  Parameters
819  ----------
820  dcrModels : `lsst.pipe.tasks.DcrModel`
821  Best fit model of the true sky after correcting chromatic effects.
822  exposure : `lsst.afw.image.ExposureF`
823  The input warped exposure to evaluate.
824  significanceImage : `numpy.ndarray`
825  Array of weights for each pixel corresponding to its significance
826  for the convergence calculation.
827  statsCtrl : `lsst.afw.math.StatisticsControl`
828  Statistics control object for coadd
829 
830  Returns
831  -------
832  convergenceMetric : `float`
833  Quality of fit metric for one exposure, within the sub-region.
834  """
835  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
836  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
837  order=self.config.imageInterpOrder,
838  splitSubfilters=self.config.splitSubfilters,
839  splitThreshold=self.config.splitThreshold,
840  amplifyModel=self.config.accelerateModel)
841  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
842  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
843 
844  finitePixels = np.isfinite(diffVals)
845  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
846  convergeMaskPixels = exposure.mask.array & convergeMask > 0
847  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
848  if np.sum(usePixels) == 0:
849  metric = 0.
850  else:
851  diffUse = diffVals[usePixels]
852  refUse = refVals[usePixels]
853  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
854  return metric
855 
856  def stackCoadd(self, dcrCoadds):
857  """Add a list of sub-band coadds together.
858 
859  Parameters
860  ----------
861  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
862  A list of coadd exposures, each exposure containing
863  the model for one subfilter.
864 
865  Returns
866  -------
867  coaddExposure : `lsst.afw.image.ExposureF`
868  A single coadd exposure that is the sum of the sub-bands.
869  """
870  coaddExposure = dcrCoadds[0].clone()
871  for coadd in dcrCoadds[1:]:
872  coaddExposure.maskedImage += coadd.maskedImage
873  return coaddExposure
874 
875  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
876  mask=None, variance=None):
877  """Create a list of coadd exposures from a list of masked images.
878 
879  Parameters
880  ----------
881  dcrModels : `lsst.pipe.tasks.DcrModel`
882  Best fit model of the true sky after correcting chromatic effects.
883  skyInfo : `lsst.pipe.base.Struct`
884  Patch geometry information, from getSkyInfo
885  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
886  The data references to the input warped exposures.
887  weightList : `list` of `float`
888  The weight to give each input exposure in the coadd
889  calibration : `lsst.afw.Image.PhotoCalib`, optional
890  Scale factor to set the photometric calibration of an exposure.
891  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
892  A record of the observations that are included in the coadd.
893  mask : `lsst.afw.image.Mask`, optional
894  Optional mask to override the values in the final coadd.
895  variance : `lsst.afw.image.Image`, optional
896  Optional variance plane to override the values in the final coadd.
897 
898  Returns
899  -------
900  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
901  A list of coadd exposures, each exposure containing
902  the model for one subfilter.
903  """
904  dcrCoadds = []
905  refModel = dcrModels.getReferenceImage()
906  for model in dcrModels:
907  if self.config.accelerateModel > 1:
908  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
909  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
910  if calibration is not None:
911  coaddExposure.setPhotoCalib(calibration)
912  if coaddInputs is not None:
913  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
914  # Set the metadata for the coadd, including PSF and aperture corrections.
915  self.assembleMetadata(coaddExposure, warpRefList, weightList)
916  # Overwrite the PSF
917  coaddExposure.setPsf(dcrModels.psf)
918  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
919  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
920  maskedImage.image = model
921  maskedImage.mask = dcrModels.mask
922  maskedImage.variance = dcrModels.variance
923  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
924  if mask is not None:
925  coaddExposure.setMask(mask)
926  if variance is not None:
927  coaddExposure.setVariance(variance)
928  dcrCoadds.append(coaddExposure)
929  return dcrCoadds
930 
931  def calculateGain(self, convergenceList, gainList):
932  """Calculate the gain to use for the current iteration.
933 
934  After calculating a new DcrModel, each value is averaged with the
935  value in the corresponding pixel from the previous iteration. This
936  reduces oscillating solutions that iterative techniques are plagued by,
937  and speeds convergence. By far the biggest changes to the model
938  happen in the first couple iterations, so we can also use a more
939  aggressive gain later when the model is changing slowly.
940 
941  Parameters
942  ----------
943  convergenceList : `list` of `float`
944  The quality of fit metric from each previous iteration.
945  gainList : `list` of `float`
946  The gains used in each previous iteration: appended with the new
947  gain value.
948  Gains are numbers between ``self.config.baseGain`` and 1.
949 
950  Returns
951  -------
952  gain : `float`
953  Relative weight to give the new solution when updating the model.
954  A value of 1.0 gives equal weight to both solutions.
955 
956  Raises
957  ------
958  ValueError
959  If ``len(convergenceList) != len(gainList)+1``.
960  """
961  nIter = len(convergenceList)
962  if nIter != len(gainList) + 1:
963  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
964  % (len(convergenceList), len(gainList)))
965 
966  if self.config.baseGain is None:
967  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
968  # The more subfilters being modeled, the lower the gain should be.
969  baseGain = 1./(self.config.dcrNumSubfilters - 1)
970  else:
971  baseGain = self.config.baseGain
972 
973  if self.config.useProgressiveGain and nIter > 2:
974  # To calculate the best gain to use, compare the past gains that have been used
975  # with the resulting convergences to estimate the best gain to use.
976  # Algorithmically, this is a Kalman filter.
977  # If forward modeling proceeds perfectly, the convergence metric should
978  # asymptotically approach a final value.
979  # We can estimate that value from the measured changes in convergence
980  # weighted by the gains used in each previous iteration.
981  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
982  for i in range(nIter - 1)]
983  # The convergence metric is strictly positive, so if the estimated final convergence is
984  # less than zero, force it to zero.
985  estFinalConv = np.array(estFinalConv)
986  estFinalConv[estFinalConv < 0] = 0
987  # Because the estimate may slowly change over time, only use the most recent measurements.
988  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
989  lastGain = gainList[-1]
990  lastConv = convergenceList[-2]
991  newConv = convergenceList[-1]
992  # The predicted convergence is the value we would get if the new model calculated
993  # in the previous iteration was perfect. Recall that the updated model that is
994  # actually used is the gain-weighted average of the new and old model,
995  # so the convergence would be similarly weighted.
996  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
997  # If the measured and predicted convergence are very close, that indicates
998  # that our forward model is accurate and we can use a more aggressive gain
999  # If the measured convergence is significantly worse (or better!) than predicted,
1000  # that indicates that the model is not converging as expected and
1001  # we should use a more conservative gain.
1002  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1003  newGain = 1 - abs(delta)
1004  # Average the gains to prevent oscillating solutions.
1005  newGain = (newGain + lastGain)/2.
1006  gain = max(baseGain, newGain)
1007  else:
1008  gain = baseGain
1009  gainList.append(gain)
1010  return gain
1011 
1012  def calculateModelWeights(self, dcrModels, dcrBBox):
1013  """Build an array that smoothly tapers to 0 away from detected sources.
1014 
1015  Parameters
1016  ----------
1017  dcrModels : `lsst.pipe.tasks.DcrModel`
1018  Best fit model of the true sky after correcting chromatic effects.
1019  dcrBBox : `lsst.geom.box.Box2I`
1020  Sub-region of the coadd which includes a buffer to allow for DCR.
1021 
1022  Returns
1023  -------
1024  weights : `numpy.ndarray` or `float`
1025  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1026  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1027 
1028  Raises
1029  ------
1030  ValueError
1031  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1032  """
1033  if not self.config.useModelWeights:
1034  return 1.0
1035  if self.config.modelWeightsWidth < 0:
1036  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1037  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1038  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1039  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1040  weights[convergeMaskPixels] = 1.
1041  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1042  weights /= np.max(weights)
1043  return weights
1044 
1045  def applyModelWeights(self, modelImages, refImage, modelWeights):
1046  """Smoothly replace model pixel values with those from a
1047  reference at locations away from detected sources.
1048 
1049  Parameters
1050  ----------
1051  modelImages : `list` of `lsst.afw.image.Image`
1052  The new DCR model images from the current iteration.
1053  The values will be modified in place.
1054  refImage : `lsst.afw.image.MaskedImage`
1055  A reference image used to supply the default pixel values.
1056  modelWeights : `numpy.ndarray` or `float`
1057  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1058  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1059  """
1060  if self.config.useModelWeights:
1061  for model in modelImages:
1062  model.array *= modelWeights
1063  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1064 
1065  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1066  """Pre-load sub-regions of a list of exposures.
1067 
1068  Parameters
1069  ----------
1070  bbox : `lsst.geom.box.Box2I`
1071  Sub-region to coadd
1072  statsCtrl : `lsst.afw.math.StatisticsControl`
1073  Statistics control object for coadd
1074  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1075  The data references to the input warped exposures.
1076  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1077  The image scalars correct for the zero point of the exposures.
1078  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1079  Each element is dict with keys = mask plane name to add the spans to
1080 
1081  Returns
1082  -------
1083  subExposures : `dict`
1084  The `dict` keys are the visit IDs,
1085  and the values are `lsst.afw.image.ExposureF`
1086  The pre-loaded exposures for the current subregion.
1087  The variance plane contains weights, and not the variance
1088  """
1089  tempExpName = self.getTempExpDatasetName(self.warpType)
1090  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1091  subExposures = {}
1092  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1093  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1094  if altMaskSpans is not None:
1095  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1096  imageScaler.scaleMaskedImage(exposure.maskedImage)
1097  # Note that the variance plane here is used to store weights, not the actual variance
1098  exposure.variance.array[:, :] = 0.
1099  # Set the weight of unmasked pixels to 1.
1100  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1101  # Set the image value of masked pixels to zero.
1102  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1103  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1104  subExposures[warpExpRef.dataId["visit"]] = exposure
1105  return subExposures
1106 
1107  def selectCoaddPsf(self, templateCoadd, warpRefList):
1108  """Compute the PSF of the coadd from the exposures with the best seeing.
1109 
1110  Parameters
1111  ----------
1112  templateCoadd : `lsst.afw.image.ExposureF`
1113  The initial coadd exposure before accounting for DCR.
1114  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1115  The data references to the input warped exposures.
1116 
1117  Returns
1118  -------
1119  psf : `lsst.meas.algorithms.CoaddPsf`
1120  The average PSF of the input exposures with the best seeing.
1121  """
1122  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1123  tempExpName = self.getTempExpDatasetName(self.warpType)
1124  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1125  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1126  psfSizeList = []
1127  for visitNum, warpExpRef in enumerate(warpRefList):
1128  psf = warpExpRef.get(tempExpName).getPsf()
1129  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1130  psfSizeList.append(psfSize)
1131  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1132  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1133  # of the mean or median FWHM of the input exposures.
1134  sizeThreshold = min(np.median(psfSizeList), psfRefSize)
1135  goodVisits = np.array(psfSizeList) <= sizeThreshold
1136  psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1137  self.config.coaddPsf.makeControl())
1138  return psf
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:134
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:165
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)