lsst.pipe.tasks  17.0.1-27-g9f381ddd+2
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.afw.geom as afwGeom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31 import lsst.meas.algorithms as measAlg
32 from lsst.meas.base import SingleFrameMeasurementTask
33 import lsst.pex.config as pexConfig
34 import lsst.pipe.base as pipeBase
35 from .assembleCoadd import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf import MeasurePsfTask
37 
38 __all__ = ["DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
39 
40 
42  dcrNumSubfilters = pexConfig.Field(
43  dtype=int,
44  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
45  default=3,
46  )
47  maxNumIter = pexConfig.Field(
48  dtype=int,
49  optional=True,
50  doc="Maximum number of iterations of forward modeling.",
51  default=None,
52  )
53  minNumIter = pexConfig.Field(
54  dtype=int,
55  optional=True,
56  doc="Minimum number of iterations of forward modeling.",
57  default=None,
58  )
59  convergenceThreshold = pexConfig.Field(
60  dtype=float,
61  doc="Target relative change in convergence between iterations of forward modeling.",
62  default=0.001,
63  )
64  useConvergence = pexConfig.Field(
65  dtype=bool,
66  doc="Use convergence test as a forward modeling end condition?"
67  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
68  default=True,
69  )
70  baseGain = pexConfig.Field(
71  dtype=float,
72  optional=True,
73  doc="Relative weight to give the new solution vs. the last solution when updating the model."
74  "A value of 1.0 gives equal weight to both solutions."
75  "Small values imply slower convergence of the solution, but can "
76  "help prevent overshooting and failures in the fit."
77  "If ``baseGain`` is None, a conservative gain "
78  "will be calculated from the number of subfilters. ",
79  default=None,
80  )
81  useProgressiveGain = pexConfig.Field(
82  dtype=bool,
83  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
84  "When calculating the next gain, we use up to 5 previous gains and convergence values."
85  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
86  default=True,
87  )
88  doAirmassWeight = pexConfig.Field(
89  dtype=bool,
90  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
91  default=False,
92  )
93  modelWeightsWidth = pexConfig.Field(
94  dtype=float,
95  doc="Width of the region around detected sources to include in the DcrModel.",
96  default=3,
97  )
98  useModelWeights = pexConfig.Field(
99  dtype=bool,
100  doc="Width of the region around detected sources to include in the DcrModel.",
101  default=True,
102  )
103  splitSubfilters = pexConfig.Field(
104  dtype=bool,
105  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
106  "Instead of at the midpoint",
107  default=True,
108  )
109  regularizeModelIterations = pexConfig.Field(
110  dtype=float,
111  doc="Maximum relative change of the model allowed between iterations."
112  "Set to zero to disable.",
113  default=2.,
114  )
115  regularizeModelFrequency = pexConfig.Field(
116  dtype=float,
117  doc="Maximum relative change of the model allowed between subfilters."
118  "Set to zero to disable.",
119  default=4.,
120  )
121  convergenceMaskPlanes = pexConfig.ListField(
122  dtype=str,
123  default=["DETECTED"],
124  doc="Mask planes to use to calculate convergence."
125  )
126  regularizationWidth = pexConfig.Field(
127  dtype=int,
128  default=2,
129  doc="Minimum radius of a region to include in regularization, in pixels."
130  )
131  imageInterpOrder = pexConfig.Field(
132  dtype=int,
133  doc="The order of the spline interpolation used to shift the image plane.",
134  default=3,
135  )
136  accelerateModel = pexConfig.Field(
137  dtype=float,
138  doc="Factor to amplify the differences between model planes by to speed convergence.",
139  default=3,
140  )
141  doCalculatePsf = pexConfig.Field(
142  dtype=bool,
143  doc="Set to detect stars and recalculate the PSF from the final coadd."
144  "Otherwise the PSF is estimated from a selection of the best input exposures",
145  default=True,
146  )
147  detectPsfSources = pexConfig.ConfigurableField(
148  target=measAlg.SourceDetectionTask,
149  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
150  )
151  measurePsfSources = pexConfig.ConfigurableField(
152  target=SingleFrameMeasurementTask,
153  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
154  )
155  measurePsf = pexConfig.ConfigurableField(
156  target=MeasurePsfTask,
157  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
158  )
159 
160  def setDefaults(self):
161  CompareWarpAssembleCoaddConfig.setDefaults(self)
162  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
163  self.doNImage = True
164  self.warpType = "direct"
165  self.assembleStaticSkyModel.warpType = self.warpType
166  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
167  self.assembleStaticSkyModel.doNImage = False
168  self.assembleStaticSkyModel.doWrite = False
169  self.detectPsfSources.returnOriginalFootprints = False
170  self.detectPsfSources.thresholdPolarity = "positive"
171  # Only use bright sources for PSF measurement
172  self.detectPsfSources.thresholdValue = 50
173  self.detectPsfSources.nSigmaToGrow = 2
174  # A valid star for PSF measurement should at least fill 5x5 pixels
175  self.detectPsfSources.minPixels = 25
176  # Use the variance plane to calculate signal to noise
177  self.detectPsfSources.thresholdType = "pixel_stdev"
178  # The signal to noise limit is good enough, while the flux limit is set
179  # in dimensionless units and may not be appropriate for all data sets.
180  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
181 
182 
184  """Assemble DCR coadded images from a set of warps.
185 
186  Attributes
187  ----------
188  bufferSize : `int`
189  The number of pixels to grow each subregion by to allow for DCR.
190 
191  Notes
192  -----
193  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
194  Warps (also called coadded temporary exposures), including the effects of
195  Differential Chromatic Refraction (DCR).
196  For full details of the mathematics and algorithm, please see
197  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
198 
199  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
200  each subfilter used in the iterative calculation.
201  It begins by dividing the bandpass-defining filter into N equal bandwidth
202  "subfilters", and divides the flux in each pixel from an initial coadd
203  equally into each as a "dcrModel". Because the airmass and parallactic
204  angle of each individual exposure is known, we can calculate the shift
205  relative to the center of the band in each subfilter due to DCR. For each
206  exposure we apply this shift as a linear transformation to the dcrModels
207  and stack the results to produce a DCR-matched exposure. The matched
208  exposures are subtracted from the input exposures to produce a set of
209  residual images, and these residuals are reverse shifted for each
210  exposures' subfilters and stacked. The shifted and stacked residuals are
211  added to the dcrModels to produce a new estimate of the flux in each pixel
212  within each subfilter. The dcrModels are solved for iteratively, which
213  continues until the solution from a new iteration improves by less than
214  a set percentage, or a maximum number of iterations is reached.
215  Two forms of regularization are employed to reduce unphysical results.
216  First, the new solution is averaged with the solution from the previous
217  iteration, which mitigates oscillating solutions where the model
218  overshoots with alternating very high and low values.
219  Second, a common degeneracy when the data have a limited range of airmass or
220  parallactic angle values is for one subfilter to be fit with very low or
221  negative values, while another subfilter is fit with very high values. This
222  typically appears in the form of holes next to sources in one subfilter,
223  and corresponding extended wings in another. Because each subfilter has
224  a narrow bandwidth we assume that physical sources that are above the noise
225  level will not vary in flux by more than a factor of `frequencyClampFactor`
226  between subfilters, and pixels that have flux deviations larger than that
227  factor will have the excess flux distributed evenly among all subfilters.
228  """
229 
230  ConfigClass = DcrAssembleCoaddConfig
231  _DefaultName = "dcrAssembleCoadd"
232 
233  def __init__(self, *args, **kwargs):
234  super().__init__(*args, **kwargs)
235  if self.config.doCalculatePsf:
236  self.schema = afwTable.SourceTable.makeMinimalSchema()
237  self.makeSubtask("detectPsfSources", schema=self.schema)
238  self.makeSubtask("measurePsfSources", schema=self.schema)
239  self.makeSubtask("measurePsf", schema=self.schema)
240 
241  @pipeBase.timeMethod
242  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
243  """Assemble a coadd from a set of warps.
244 
245  Coadd a set of Warps. Compute weights to be applied to each Warp and
246  find scalings to match the photometric zeropoint to a reference Warp.
247  Assemble the Warps using run method.
248  Forward model chromatic effects across multiple subfilters,
249  and subtract from the input Warps to build sets of residuals.
250  Use the residuals to construct a new ``DcrModel`` for each subfilter,
251  and iterate until the model converges.
252  Interpolate over NaNs and optionally write the coadd to disk.
253  Return the coadded exposure.
254 
255  Parameters
256  ----------
257  dataRef : `lsst.daf.persistence.ButlerDataRef`
258  Data reference defining the patch for coaddition and the
259  reference Warp
260  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
261  List of data references to warps. Data to be coadded will be
262  selected from this list based on overlap with the patch defined by
263  the data reference.
264 
265  Returns
266  -------
267  results : `lsst.pipe.base.Struct`
268  The Struct contains the following fields:
269 
270  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
271  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
272  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
273  - ``dcrNImages``: `list` of exposure count images for each subfilter
274  """
275  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
276  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
277 
278  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
279  warpRefList=warpRefList)
280  if results is None:
281  skyInfo = self.getSkyInfo(dataRef)
282  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
283  skyInfo.patchInfo.getIndex())
284  return
285  for subfilter in range(self.config.dcrNumSubfilters):
286  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
287  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
288  AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
289  if self.config.doWrite:
290  self.log.info("Persisting dcrCoadd")
291  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
292  numSubfilters=self.config.dcrNumSubfilters)
293  if self.config.doNImage and results.dcrNImages is not None:
294  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
295  numSubfilters=self.config.dcrNumSubfilters)
296 
297  return results
298 
299  def processResults(self, coaddExposure, dataRef):
300  """Interpolate over missing data and mask bright stars.
301 
302  Also detect sources on the coadd exposure and measure the final PSF,
303  if ``doCalculatePsf`` is set.
304 
305  Parameters
306  ----------
307  coaddExposure : `lsst.afw.image.Exposure`
308  The final coadded exposure.
309  dataRef : `lsst.daf.persistence.ButlerDataRef`
310  Data reference defining the patch for coaddition and the
311  reference Warp
312  """
313  super().processResults(coaddExposure, dataRef)
314 
315  if self.config.doCalculatePsf:
316  expId = dataRef.get("dcrCoaddId")
317  table = afwTable.SourceTable.make(self.schema)
318  detResults = self.detectPsfSources.run(table, coaddExposure, expId, clearMask=False)
319  coaddSources = detResults.sources
320  self.measurePsfSources.run(
321  measCat=coaddSources,
322  exposure=coaddExposure,
323  exposureId=expId
324  )
325  try:
326  psfResults = self.measurePsf.run(coaddExposure, coaddSources, expId=expId)
327  except RuntimeError as e:
328  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
329  else:
330  coaddExposure.setPsf(psfResults.psf)
331 
332  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
333  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
334 
335  Sets the property ``bufferSize``.
336 
337  Parameters
338  ----------
339  templateCoadd : `lsst.afw.image.ExposureF`
340  The initial coadd exposure before accounting for DCR.
341  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
342  The data references to the input warped exposures.
343  weightList : `list` of `float`
344  The weight to give each input exposure in the coadd
345  Will be modified in place if ``doAirmassWeight`` is set.
346 
347  Returns
348  -------
349  dcrModels : `lsst.pipe.tasks.DcrModel`
350  Best fit model of the true sky after correcting chromatic effects.
351 
352  Raises
353  ------
354  NotImplementedError
355  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
356  """
357  filterInfo = templateCoadd.getFilter()
358  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
359  raise NotImplementedError("No minimum/maximum wavelength information found"
360  " in the filter definition! Please add lambdaMin and lambdaMax"
361  " to the Mapper class in your obs package.")
362  tempExpName = self.getTempExpDatasetName(self.warpType)
363  dcrShifts = []
364  airmassDict = {}
365  angleDict = {}
366  for visitNum, warpExpRef in enumerate(warpRefList):
367  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
368  visit = warpExpRef.dataId["visit"]
369  airmass = visitInfo.getBoresightAirmass()
370  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
371  airmassDict[visit] = airmass
372  angleDict[visit] = parallacticAngle
373  if self.config.doAirmassWeight:
374  weightList[visitNum] *= airmass
375  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
376  filterInfo, self.config.dcrNumSubfilters))))
377  self.log.info("Selected airmasses:\n%s", airmassDict)
378  self.log.info("Selected parallactic angles:\n%s", angleDict)
379  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
380  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
381  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
382  self.config.dcrNumSubfilters,
383  filterInfo=filterInfo,
384  psf=psf)
385  return dcrModels
386 
387  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
388  supplementaryData=None):
389  """Assemble the coadd.
390 
391  Requires additional inputs Struct ``supplementaryData`` to contain a
392  ``templateCoadd`` that serves as the model of the static sky.
393 
394  Find artifacts and apply them to the warps' masks creating a list of
395  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
396  Then pass these alternative masks to the base class's assemble method.
397 
398  Divide the ``templateCoadd`` evenly between each subfilter of a
399  ``DcrModel`` as the starting best estimate of the true wavelength-
400  dependent sky. Forward model the ``DcrModel`` using the known
401  chromatic effects in each subfilter and calculate a convergence metric
402  based on how well the modeled template matches the input warps. If
403  the convergence has not yet reached the desired threshold, then shift
404  and stack the residual images to build a new ``DcrModel``. Apply
405  conditioning to prevent oscillating solutions between iterations or
406  between subfilters.
407 
408  Once the ``DcrModel`` reaches convergence or the maximum number of
409  iterations has been reached, fill the metadata for each subfilter
410  image and make them proper ``coaddExposure``s.
411 
412  Parameters
413  ----------
414  skyInfo : `lsst.pipe.base.Struct`
415  Patch geometry information, from getSkyInfo
416  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
417  The data references to the input warped exposures.
418  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
419  The image scalars correct for the zero point of the exposures.
420  weightList : `list` of `float`
421  The weight to give each input exposure in the coadd
422  supplementaryData : `lsst.pipe.base.Struct`
423  Result struct returned by ``makeSupplementaryData`` with components:
424 
425  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
426 
427  Returns
428  -------
429  result : `lsst.pipe.base.Struct`
430  Result struct with components:
431 
432  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
433  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
434  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
435  - ``dcrNImages``: `list` of exposure count images for each subfilter
436  """
437  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
438  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
439  templateCoadd = supplementaryData.templateCoadd
440  baseMask = templateCoadd.mask.clone()
441  # The variance plane is for each subfilter
442  # and should be proportionately lower than the full-band image
443  baseVariance = templateCoadd.variance.clone()
444  baseVariance /= self.config.dcrNumSubfilters
445  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
446  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
447  templateCoadd.setMask(baseMask)
448  badMaskPlanes = self.config.badMaskPlanes[:]
449  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
450  # This is because pixels in observations that are significantly affect by DCR
451  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
452  # but those are necessary to constrain the DCR model.
453  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
454 
455  stats = self.prepareStats(mask=badPixelMask)
456  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
457  if self.config.doNImage:
458  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
459  spanSetMaskList, stats.ctrl)
460  nImage = afwImage.ImageU(skyInfo.bbox)
461  # Note that this nImage will be a factor of dcrNumSubfilters higher than
462  # the nImage returned by assembleCoadd for most pixels. This is because each
463  # subfilter may have a different nImage, and fractional values are not allowed.
464  for dcrNImage in dcrNImages:
465  nImage += dcrNImage
466  else:
467  dcrNImages = None
468 
469  subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
470  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
471  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
472  subIter = 0
473  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
474  modelIter = 0
475  subIter += 1
476  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
477  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
478  dcrBBox = afwGeom.Box2I(subBBox)
479  dcrBBox.grow(self.bufferSize)
480  dcrBBox.clip(dcrModels.bbox)
481  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
482  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
483  imageScalerList, spanSetMaskList)
484  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
485  warpRefList, weightList, stats.ctrl)
486  self.log.info("Initial convergence : %s", convergenceMetric)
487  convergenceList = [convergenceMetric]
488  gainList = []
489  convergenceCheck = 1.
490  refImage = templateCoadd.image
491  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
492  gain = self.calculateGain(convergenceList, gainList)
493  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
494  stats.ctrl, convergenceMetric, gain,
495  modelWeights, refImage, dcrWeights)
496  if self.config.useConvergence:
497  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
498  warpRefList, weightList, stats.ctrl)
499  if convergenceMetric == 0:
500  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
501  "most likely due to there being no valid data in the region.",
502  skyInfo.patchInfo.getIndex(), subIter)
503  break
504  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
505  if (convergenceCheck < 0) & (modelIter > minNumIter):
506  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
507  "iterations or desired convergence improvement of %s."
508  " Divergence: %s",
509  skyInfo.patchInfo.getIndex(), subIter,
510  self.config.convergenceThreshold, convergenceCheck)
511  break
512  convergenceList.append(convergenceMetric)
513  if modelIter > maxNumIter:
514  if self.config.useConvergence:
515  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
516  "before reaching desired convergence improvement of %s."
517  " Final convergence improvement: %s",
518  skyInfo.patchInfo.getIndex(), subIter,
519  self.config.convergenceThreshold, convergenceCheck)
520  break
521 
522  if self.config.useConvergence:
523  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
524  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
525  modelIter += 1
526  else:
527  if self.config.useConvergence:
528  self.log.info("Coadd patch %s subregion %s finished with "
529  "convergence metric %s after %s iterations",
530  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
531  else:
532  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
533  skyInfo.patchInfo.getIndex(), subIter, modelIter)
534  if self.config.useConvergence and convergenceMetric > 0:
535  self.log.info("Final convergence improvement was %.4f%% overall",
536  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
537 
538  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
539  calibration=self.scaleZeroPoint.getPhotoCalib(),
540  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
541  mask=baseMask,
542  variance=baseVariance)
543  coaddExposure = self.stackCoadd(dcrCoadds)
544  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
545  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
546 
547  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
548  """Calculate the number of exposures contributing to each subfilter.
549 
550  Parameters
551  ----------
552  dcrModels : `lsst.pipe.tasks.DcrModel`
553  Best fit model of the true sky after correcting chromatic effects.
554  bbox : `lsst.afw.geom.box.Box2I`
555  Bounding box of the patch to coadd.
556  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
557  The data references to the input warped exposures.
558  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
559  Each element of the `dict` contains the new mask plane name
560  (e.g. "CLIPPED and/or "NO_DATA") as the key,
561  and the list of SpanSets to apply to the mask.
562  statsCtrl : `lsst.afw.math.StatisticsControl`
563  Statistics control object for coadd
564 
565  Returns
566  -------
567  dcrNImages : `list` of `lsst.afw.image.ImageU`
568  List of exposure count images for each subfilter
569  dcrWeights : `list` of `lsst.afw.image.ImageF`
570  Per-pixel weights for each subfilter.
571  Equal to 1/(number of unmasked images contributing to each pixel).
572  """
573  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
574  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
575  tempExpName = self.getTempExpDatasetName(self.warpType)
576  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
577  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
578  visitInfo = exposure.getInfo().getVisitInfo()
579  wcs = exposure.getInfo().getWcs()
580  mask = exposure.mask
581  if altMaskSpans is not None:
582  self.applyAltMaskPlanes(mask, altMaskSpans)
583  weightImage = np.zeros_like(exposure.image.array)
584  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
585  dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
586  for dcr, dcrNImage, dcrWeight in zip(dcrShift, dcrNImages, dcrWeights):
587  # Note that we use the same interpolation for the weights and the images.
588  shiftedWeights = applyDcr(weightImage, dcr, useInverse=True,
589  order=self.config.imageInterpOrder)
590  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
591  dcrWeight.array += shiftedWeights
592  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
593  weightsThreshold = 1.
594  goodPix = dcrWeights[0].array > weightsThreshold
595  for weights in dcrWeights[1:]:
596  goodPix = (weights.array > weightsThreshold) & goodPix
597  for subfilter in range(self.config.dcrNumSubfilters):
598  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
599  dcrWeights[subfilter].array[~goodPix] = 0.
600  dcrNImages[subfilter].array[~goodPix] = 0
601  return (dcrNImages, dcrWeights)
602 
603  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
604  statsCtrl, convergenceMetric,
605  gain, modelWeights, refImage, dcrWeights):
606  """Assemble the DCR coadd for a sub-region.
607 
608  Build a DCR-matched template for each input exposure, then shift the
609  residuals according to the DCR in each subfilter.
610  Stack the shifted residuals and apply them as a correction to the
611  solution from the previous iteration.
612  Restrict the new model solutions from varying by more than a factor of
613  `modelClampFactor` from the last solution, and additionally restrict the
614  individual subfilter models from varying by more than a factor of
615  `frequencyClampFactor` from their average.
616  Finally, mitigate potentially oscillating solutions by averaging the new
617  solution with the solution from the previous iteration, weighted by
618  their convergence metric.
619 
620  Parameters
621  ----------
622  dcrModels : `lsst.pipe.tasks.DcrModel`
623  Best fit model of the true sky after correcting chromatic effects.
624  subExposures : `dict` of `lsst.afw.image.ExposureF`
625  The pre-loaded exposures for the current subregion.
626  bbox : `lsst.afw.geom.box.Box2I`
627  Bounding box of the subregion to coadd.
628  dcrBBox : `lsst.afw.geom.box.Box2I`
629  Sub-region of the coadd which includes a buffer to allow for DCR.
630  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
631  The data references to the input warped exposures.
632  statsCtrl : `lsst.afw.math.StatisticsControl`
633  Statistics control object for coadd
634  convergenceMetric : `float`
635  Quality of fit metric for the matched templates of the input images.
636  gain : `float`, optional
637  Relative weight to give the new solution when updating the model.
638  modelWeights : `numpy.ndarray` or `float`
639  A 2D array of weight values that tapers smoothly to zero away from detected sources.
640  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
641  refImage : `lsst.afw.image.Image`
642  A reference image used to supply the default pixel values.
643  dcrWeights : `list` of `lsst.afw.image.Image`
644  Per-pixel weights for each subfilter.
645  Equal to 1/(number of unmasked images contributing to each pixel).
646  """
647  residualGeneratorList = []
648 
649  for warpExpRef in warpRefList:
650  exposure = subExposures[warpExpRef.dataId["visit"]]
651  visitInfo = exposure.getInfo().getVisitInfo()
652  wcs = exposure.getInfo().getWcs()
653  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
654  order=self.config.imageInterpOrder,
655  splitSubfilters=self.config.splitSubfilters,
656  amplifyModel=self.config.accelerateModel)
657  residual = exposure.image.array - templateImage.array
658  # Note that the variance plane here is used to store weights, not the actual variance
659  residual *= exposure.variance.array
660  # The residuals are stored as a list of generators.
661  # This allows the residual for a given subfilter and exposure to be created
662  # on the fly, instead of needing to store them all in memory.
663  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
664 
665  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
666  gain=gain,
667  modelWeights=modelWeights,
668  refImage=refImage,
669  dcrWeights=dcrWeights)
670  dcrModels.assign(dcrSubModelOut, bbox)
671 
672  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
673  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
674 
675  Parameters
676  ----------
677  residual : `numpy.ndarray`
678  The residual masked image for one exposure,
679  after subtracting the matched template
680  visitInfo : `lsst.afw.image.VisitInfo`
681  Metadata for the exposure.
682  wcs : `lsst.afw.geom.SkyWcs`
683  Coordinate system definition (wcs) for the exposure.
684  filterInfo : `lsst.afw.image.Filter`
685  The filter definition, set in the current instruments' obs package.
686  Required for any calculation of DCR, including making matched templates.
687 
688  Yields
689  ------
690  residualImage : `numpy.ndarray`
691  The residual image for the next subfilter, shifted for DCR.
692  """
693  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
694  for dcr in dcrShift:
695  yield applyDcr(residual, dcr, useInverse=True, order=self.config.imageInterpOrder)
696 
697  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
698  gain, modelWeights, refImage, dcrWeights):
699  """Calculate a new DcrModel from a set of image residuals.
700 
701  Parameters
702  ----------
703  dcrModels : `lsst.pipe.tasks.DcrModel`
704  Current model of the true sky after correcting chromatic effects.
705  residualGeneratorList : `generator` of `numpy.ndarray`
706  The residual image for the next subfilter, shifted for DCR.
707  dcrBBox : `lsst.afw.geom.box.Box2I`
708  Sub-region of the coadd which includes a buffer to allow for DCR.
709  statsCtrl : `lsst.afw.math.StatisticsControl`
710  Statistics control object for coadd
711  gain : `float`
712  Relative weight to give the new solution when updating the model.
713  modelWeights : `numpy.ndarray` or `float`
714  A 2D array of weight values that tapers smoothly to zero away from detected sources.
715  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
716  refImage : `lsst.afw.image.Image`
717  A reference image used to supply the default pixel values.
718  dcrWeights : `list` of `lsst.afw.image.Image`
719  Per-pixel weights for each subfilter.
720  Equal to 1/(number of unmasked images contributing to each pixel).
721 
722  Returns
723  -------
724  dcrModel : `lsst.pipe.tasks.DcrModel`
725  New model of the true sky after correcting chromatic effects.
726  """
727  newModelImages = []
728  for subfilter, model in enumerate(dcrModels):
729  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
730  residual = np.sum(residualsList, axis=0)
731  residual *= dcrWeights[subfilter][dcrBBox].array
732  # `MaskedImage`s only support in-place addition, so rename for readability
733  newModel = model[dcrBBox].clone()
734  newModel.array += residual
735  # Catch any invalid values
736  badPixels = ~np.isfinite(newModel.array)
737  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
738  if self.config.regularizeModelIterations > 0:
739  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
740  self.config.regularizeModelIterations,
741  self.config.regularizationWidth)
742  newModelImages.append(newModel)
743  if self.config.regularizeModelFrequency > 0:
744  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
745  self.config.regularizeModelFrequency,
746  self.config.regularizationWidth)
747  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
748  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
749  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
750  dcrModels.mask, dcrModels.variance)
751 
752  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
753  """Calculate a quality of fit metric for the matched templates.
754 
755  Parameters
756  ----------
757  dcrModels : `lsst.pipe.tasks.DcrModel`
758  Best fit model of the true sky after correcting chromatic effects.
759  subExposures : `dict` of `lsst.afw.image.ExposureF`
760  The pre-loaded exposures for the current subregion.
761  bbox : `lsst.afw.geom.box.Box2I`
762  Sub-region to coadd
763  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
764  The data references to the input warped exposures.
765  weightList : `list` of `float`
766  The weight to give each input exposure in the coadd
767  statsCtrl : `lsst.afw.math.StatisticsControl`
768  Statistics control object for coadd
769 
770  Returns
771  -------
772  convergenceMetric : `float`
773  Quality of fit metric for all input exposures, within the sub-region
774  """
775  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
776  nSigma = 3.
777  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
778  bufferSize=self.bufferSize)
779  if np.max(significanceImage) == 0:
780  significanceImage += 1.
781  weight = 0
782  metric = 0.
783  metricList = {}
784  for warpExpRef, expWeight in zip(warpRefList, weightList):
785  exposure = subExposures[warpExpRef.dataId["visit"]][bbox]
786  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
787  metric += singleMetric
788  metricList[warpExpRef.dataId["visit"]] = singleMetric
789  weight += 1.
790  self.log.info("Individual metrics:\n%s", metricList)
791  return 1.0 if weight == 0.0 else metric/weight
792 
793  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
794  """Calculate a quality of fit metric for a single matched template.
795 
796  Parameters
797  ----------
798  dcrModels : `lsst.pipe.tasks.DcrModel`
799  Best fit model of the true sky after correcting chromatic effects.
800  exposure : `lsst.afw.image.ExposureF`
801  The input warped exposure to evaluate.
802  significanceImage : `numpy.ndarray`
803  Array of weights for each pixel corresponding to its significance
804  for the convergence calculation.
805  statsCtrl : `lsst.afw.math.StatisticsControl`
806  Statistics control object for coadd
807 
808  Returns
809  -------
810  convergenceMetric : `float`
811  Quality of fit metric for one exposure, within the sub-region.
812  """
813  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
814  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
815  order=self.config.imageInterpOrder,
816  splitSubfilters=self.config.splitSubfilters,
817  amplifyModel=self.config.accelerateModel)
818  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
819  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
820 
821  finitePixels = np.isfinite(diffVals)
822  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
823  convergeMaskPixels = exposure.mask.array & convergeMask > 0
824  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
825  if np.sum(usePixels) == 0:
826  metric = 0.
827  else:
828  diffUse = diffVals[usePixels]
829  refUse = refVals[usePixels]
830  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
831  return metric
832 
833  def stackCoadd(self, dcrCoadds):
834  """Add a list of sub-band coadds together.
835 
836  Parameters
837  ----------
838  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
839  A list of coadd exposures, each exposure containing
840  the model for one subfilter.
841 
842  Returns
843  -------
844  coaddExposure : `lsst.afw.image.ExposureF`
845  A single coadd exposure that is the sum of the sub-bands.
846  """
847  coaddExposure = dcrCoadds[0].clone()
848  for coadd in dcrCoadds[1:]:
849  coaddExposure.maskedImage += coadd.maskedImage
850  return coaddExposure
851 
852  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
853  mask=None, variance=None):
854  """Create a list of coadd exposures from a list of masked images.
855 
856  Parameters
857  ----------
858  dcrModels : `lsst.pipe.tasks.DcrModel`
859  Best fit model of the true sky after correcting chromatic effects.
860  skyInfo : `lsst.pipe.base.Struct`
861  Patch geometry information, from getSkyInfo
862  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
863  The data references to the input warped exposures.
864  weightList : `list` of `float`
865  The weight to give each input exposure in the coadd
866  calibration : `lsst.afw.Image.PhotoCalib`, optional
867  Scale factor to set the photometric calibration of an exposure.
868  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
869  A record of the observations that are included in the coadd.
870  mask : `lsst.afw.image.Mask`, optional
871  Optional mask to override the values in the final coadd.
872  variance : `lsst.afw.image.Image`, optional
873  Optional variance plane to override the values in the final coadd.
874 
875  Returns
876  -------
877  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
878  A list of coadd exposures, each exposure containing
879  the model for one subfilter.
880  """
881  dcrCoadds = []
882  refModel = dcrModels.getReferenceImage()
883  for model in dcrModels:
884  if self.config.accelerateModel > 1:
885  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
886  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
887  if calibration is not None:
888  coaddExposure.setPhotoCalib(calibration)
889  if coaddInputs is not None:
890  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
891  # Set the metadata for the coadd, including PSF and aperture corrections.
892  self.assembleMetadata(coaddExposure, warpRefList, weightList)
893  # Overwrite the PSF
894  coaddExposure.setPsf(dcrModels.psf)
895  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
896  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
897  maskedImage.image = model
898  maskedImage.mask = dcrModels.mask
899  maskedImage.variance = dcrModels.variance
900  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
901  if mask is not None:
902  coaddExposure.setMask(mask)
903  if variance is not None:
904  coaddExposure.setVariance(variance)
905  dcrCoadds.append(coaddExposure)
906  return dcrCoadds
907 
908  def calculateGain(self, convergenceList, gainList):
909  """Calculate the gain to use for the current iteration.
910 
911  After calculating a new DcrModel, each value is averaged with the
912  value in the corresponding pixel from the previous iteration. This
913  reduces oscillating solutions that iterative techniques are plagued by,
914  and speeds convergence. By far the biggest changes to the model
915  happen in the first couple iterations, so we can also use a more
916  aggressive gain later when the model is changing slowly.
917 
918  Parameters
919  ----------
920  convergenceList : `list` of `float`
921  The quality of fit metric from each previous iteration.
922  gainList : `list` of `float`
923  The gains used in each previous iteration: appended with the new
924  gain value.
925  Gains are numbers between ``self.config.baseGain`` and 1.
926 
927  Returns
928  -------
929  gain : `float`
930  Relative weight to give the new solution when updating the model.
931  A value of 1.0 gives equal weight to both solutions.
932 
933  Raises
934  ------
935  ValueError
936  If ``len(convergenceList) != len(gainList)+1``.
937  """
938  nIter = len(convergenceList)
939  if nIter != len(gainList) + 1:
940  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
941  % (len(convergenceList), len(gainList)))
942 
943  if self.config.baseGain is None:
944  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
945  # The more subfilters being modeled, the lower the gain should be.
946  baseGain = 1./(self.config.dcrNumSubfilters - 1)
947  else:
948  baseGain = self.config.baseGain
949 
950  if self.config.useProgressiveGain and nIter > 2:
951  # To calculate the best gain to use, compare the past gains that have been used
952  # with the resulting convergences to estimate the best gain to use.
953  # Algorithmically, this is a Kalman filter.
954  # If forward modeling proceeds perfectly, the convergence metric should
955  # asymptotically approach a final value.
956  # We can estimate that value from the measured changes in convergence
957  # weighted by the gains used in each previous iteration.
958  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
959  for i in range(nIter - 1)]
960  # The convergence metric is strictly positive, so if the estimated final convergence is
961  # less than zero, force it to zero.
962  estFinalConv = np.array(estFinalConv)
963  estFinalConv[estFinalConv < 0] = 0
964  # Because the estimate may slowly change over time, only use the most recent measurements.
965  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
966  lastGain = gainList[-1]
967  lastConv = convergenceList[-2]
968  newConv = convergenceList[-1]
969  # The predicted convergence is the value we would get if the new model calculated
970  # in the previous iteration was perfect. Recall that the updated model that is
971  # actually used is the gain-weighted average of the new and old model,
972  # so the convergence would be similarly weighted.
973  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
974  # If the measured and predicted convergence are very close, that indicates
975  # that our forward model is accurate and we can use a more aggressive gain
976  # If the measured convergence is significantly worse (or better!) than predicted,
977  # that indicates that the model is not converging as expected and
978  # we should use a more conservative gain.
979  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
980  newGain = 1 - abs(delta)
981  # Average the gains to prevent oscillating solutions.
982  newGain = (newGain + lastGain)/2.
983  gain = max(baseGain, newGain)
984  else:
985  gain = baseGain
986  gainList.append(gain)
987  return gain
988 
989  def calculateModelWeights(self, dcrModels, dcrBBox):
990  """Build an array that smoothly tapers to 0 away from detected sources.
991 
992  Parameters
993  ----------
994  dcrModels : `lsst.pipe.tasks.DcrModel`
995  Best fit model of the true sky after correcting chromatic effects.
996  dcrBBox : `lsst.afw.geom.box.Box2I`
997  Sub-region of the coadd which includes a buffer to allow for DCR.
998 
999  Returns
1000  -------
1001  weights : `numpy.ndarray` or `float`
1002  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1003  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1004 
1005  Raises
1006  ------
1007  ValueError
1008  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1009  """
1010  if not self.config.useModelWeights:
1011  return 1.0
1012  if self.config.modelWeightsWidth < 0:
1013  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1014  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1015  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1016  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1017  weights[convergeMaskPixels] = 1.
1018  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1019  weights /= np.max(weights)
1020  return weights
1021 
1022  def applyModelWeights(self, modelImages, refImage, modelWeights):
1023  """Smoothly replace model pixel values with those from a
1024  reference at locations away from detected sources.
1025 
1026  Parameters
1027  ----------
1028  modelImages : `list` of `lsst.afw.image.Image`
1029  The new DCR model images from the current iteration.
1030  The values will be modified in place.
1031  refImage : `lsst.afw.image.MaskedImage`
1032  A reference image used to supply the default pixel values.
1033  modelWeights : `numpy.ndarray` or `float`
1034  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1035  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1036  """
1037  if self.config.useModelWeights:
1038  for model in modelImages:
1039  model.array *= modelWeights
1040  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1041 
1042  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1043  """Pre-load sub-regions of a list of exposures.
1044 
1045  Parameters
1046  ----------
1047  bbox : `lsst.afw.geom.box.Box2I`
1048  Sub-region to coadd
1049  statsCtrl : `lsst.afw.math.StatisticsControl`
1050  Statistics control object for coadd
1051  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1052  The data references to the input warped exposures.
1053  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1054  The image scalars correct for the zero point of the exposures.
1055  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1056  Each element is dict with keys = mask plane name to add the spans to
1057 
1058  Returns
1059  -------
1060  subExposures : `dict`
1061  The `dict` keys are the visit IDs,
1062  and the values are `lsst.afw.image.ExposureF`
1063  The pre-loaded exposures for the current subregion.
1064  The variance plane contains weights, and not the variance
1065  """
1066  tempExpName = self.getTempExpDatasetName(self.warpType)
1067  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1068  subExposures = {}
1069  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1070  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1071  if altMaskSpans is not None:
1072  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1073  imageScaler.scaleMaskedImage(exposure.maskedImage)
1074  # Note that the variance plane here is used to store weights, not the actual variance
1075  exposure.variance.array[:, :] = 0.
1076  # Set the weight of unmasked pixels to 1.
1077  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1078  # Set the image value of masked pixels to zero.
1079  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1080  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1081  subExposures[warpExpRef.dataId["visit"]] = exposure
1082  return subExposures
1083 
1084  def selectCoaddPsf(self, templateCoadd, warpRefList):
1085  """Compute the PSF of the coadd from the exposures with the best seeing.
1086 
1087  Parameters
1088  ----------
1089  templateCoadd : `lsst.afw.image.ExposureF`
1090  The initial coadd exposure before accounting for DCR.
1091  warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef`
1092  The data references to the input warped exposures.
1093 
1094  Returns
1095  -------
1096  psf : `lsst.meas.algorithms.CoaddPsf`
1097  The average PSF of the input exposures with the best seeing.
1098  """
1099  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1100  tempExpName = self.getTempExpDatasetName(self.warpType)
1101  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1102  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1103  psfSizeList = []
1104  for visitNum, warpExpRef in enumerate(warpRefList):
1105  psf = warpExpRef.get(tempExpName).getPsf()
1106  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1107  psfSizeList.append(psfSize)
1108  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1109  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1110  # of the mean or median FWHM of the input exposures.
1111  sizeThreshold = min(np.median(psfSizeList), psfRefSize)
1112  goodVisits = np.array(psfSizeList) <= sizeThreshold
1113  psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1114  self.config.coaddPsf.makeControl())
1115  return psf
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
Definition: coaddBase.py:133
def getTempExpDatasetName(self, warpType="direct")
Definition: coaddBase.py:164
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)