lsst.pipe.tasks  19.0.0-28-g53bcf5f6+3
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32 import lsst.meas.algorithms as measAlg
33 from lsst.meas.base import SingleFrameMeasurementTask
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 import lsst.utils as utils
37 from .assembleCoadd import (AssembleCoaddTask,
38  CompareWarpAssembleCoaddConfig,
39  CompareWarpAssembleCoaddTask)
40 from .coaddBase import makeSkyInfo
41 from .measurePsf import MeasurePsfTask
42 
43 __all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
44 
45 
46 class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47  dimensions=("tract", "patch", "abstract_filter", "skymap"),
48  defaultTemplates={"inputCoaddName": "deep",
49  "outputCoaddName": "dcr",
50  "warpType": "direct",
51  "warpTypeSuffix": "",
52  "fakesType": ""}):
53  inputWarps = pipeBase.connectionTypes.Input(
54  doc=("Input list of warps to be assembled i.e. stacked."
55  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56  name="{inputCoaddName}Coadd_{warpType}Warp",
57  storageClass="ExposureF",
58  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59  deferLoad=True,
60  multiple=True
61  )
62  skyMap = pipeBase.connectionTypes.Input(
63  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64  name="{inputCoaddName}Coadd_skyMap",
65  storageClass="SkyMap",
66  dimensions=("skymap", ),
67  )
68  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70  " BRIGHT_OBJECT."),
71  name="brightObjectMask",
72  storageClass="ObjectMaskCatalog",
73  dimensions=("tract", "patch", "skymap", "abstract_filter"),
74  )
75  templateExposure = pipeBase.connectionTypes.Input(
76  doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77  name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78  storageClass="ExposureF",
79  dimensions=("tract", "patch", "skymap", "abstract_filter"),
80  )
81  dcrCoadds = pipeBase.connectionTypes.Output(
82  doc="Output coadded exposure, produced by stacking input warps",
83  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84  storageClass="ExposureF",
85  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
86  multiple=True,
87  )
88  dcrNImages = pipeBase.connectionTypes.Output(
89  doc="Output image of number of input images per pixel",
90  name="{outputCoaddName}Coadd_nImage",
91  storageClass="ImageU",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  )
95 
96  def __init__(self, *, config=None):
97  super().__init__(config=config)
98  if not config.doWrite:
99  self.outputs.remove("dcrCoadds")
100 
101 
102 class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103  pipelineConnections=DcrAssembleCoaddConnections):
104  dcrNumSubfilters = pexConfig.Field(
105  dtype=int,
106  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107  default=3,
108  )
109  maxNumIter = pexConfig.Field(
110  dtype=int,
111  optional=True,
112  doc="Maximum number of iterations of forward modeling.",
113  default=None,
114  )
115  minNumIter = pexConfig.Field(
116  dtype=int,
117  optional=True,
118  doc="Minimum number of iterations of forward modeling.",
119  default=None,
120  )
121  convergenceThreshold = pexConfig.Field(
122  dtype=float,
123  doc="Target relative change in convergence between iterations of forward modeling.",
124  default=0.001,
125  )
126  useConvergence = pexConfig.Field(
127  dtype=bool,
128  doc="Use convergence test as a forward modeling end condition?"
129  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130  default=True,
131  )
132  baseGain = pexConfig.Field(
133  dtype=float,
134  optional=True,
135  doc="Relative weight to give the new solution vs. the last solution when updating the model."
136  "A value of 1.0 gives equal weight to both solutions."
137  "Small values imply slower convergence of the solution, but can "
138  "help prevent overshooting and failures in the fit."
139  "If ``baseGain`` is None, a conservative gain "
140  "will be calculated from the number of subfilters. ",
141  default=None,
142  )
143  useProgressiveGain = pexConfig.Field(
144  dtype=bool,
145  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146  "When calculating the next gain, we use up to 5 previous gains and convergence values."
147  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148  default=True,
149  )
150  doAirmassWeight = pexConfig.Field(
151  dtype=bool,
152  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153  default=False,
154  )
155  modelWeightsWidth = pexConfig.Field(
156  dtype=float,
157  doc="Width of the region around detected sources to include in the DcrModel.",
158  default=3,
159  )
160  useModelWeights = pexConfig.Field(
161  dtype=bool,
162  doc="Width of the region around detected sources to include in the DcrModel.",
163  default=True,
164  )
165  splitSubfilters = pexConfig.Field(
166  dtype=bool,
167  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168  "Instead of at the midpoint",
169  default=True,
170  )
171  splitThreshold = pexConfig.Field(
172  dtype=float,
173  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174  "Set to 0 to always split the subfilters.",
175  default=0.1,
176  )
177  regularizeModelIterations = pexConfig.Field(
178  dtype=float,
179  doc="Maximum relative change of the model allowed between iterations."
180  "Set to zero to disable.",
181  default=2.,
182  )
183  regularizeModelFrequency = pexConfig.Field(
184  dtype=float,
185  doc="Maximum relative change of the model allowed between subfilters."
186  "Set to zero to disable.",
187  default=4.,
188  )
189  convergenceMaskPlanes = pexConfig.ListField(
190  dtype=str,
191  default=["DETECTED"],
192  doc="Mask planes to use to calculate convergence."
193  )
194  regularizationWidth = pexConfig.Field(
195  dtype=int,
196  default=2,
197  doc="Minimum radius of a region to include in regularization, in pixels."
198  )
199  imageInterpOrder = pexConfig.Field(
200  dtype=int,
201  doc="The order of the spline interpolation used to shift the image plane.",
202  default=3,
203  )
204  accelerateModel = pexConfig.Field(
205  dtype=float,
206  doc="Factor to amplify the differences between model planes by to speed convergence.",
207  default=3,
208  )
209  doCalculatePsf = pexConfig.Field(
210  dtype=bool,
211  doc="Set to detect stars and recalculate the PSF from the final coadd."
212  "Otherwise the PSF is estimated from a selection of the best input exposures",
213  default=False,
214  )
215  detectPsfSources = pexConfig.ConfigurableField(
216  target=measAlg.SourceDetectionTask,
217  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218  )
219  measurePsfSources = pexConfig.ConfigurableField(
220  target=SingleFrameMeasurementTask,
221  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222  )
223  measurePsf = pexConfig.ConfigurableField(
224  target=MeasurePsfTask,
225  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226  )
227 
228  def setDefaults(self):
229  CompareWarpAssembleCoaddConfig.setDefaults(self)
230  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
231  self.doNImage = True
232  self.warpType = "direct"
233  self.assembleStaticSkyModel.warpType = self.warpType
234  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
235  self.assembleStaticSkyModel.doNImage = False
236  self.assembleStaticSkyModel.doWrite = False
237  self.detectPsfSources.returnOriginalFootprints = False
238  self.detectPsfSources.thresholdPolarity = "positive"
239  # Only use bright sources for PSF measurement
240  self.detectPsfSources.thresholdValue = 50
241  self.detectPsfSources.nSigmaToGrow = 2
242  # A valid star for PSF measurement should at least fill 5x5 pixels
243  self.detectPsfSources.minPixels = 25
244  # Use the variance plane to calculate signal to noise
245  self.detectPsfSources.thresholdType = "pixel_stdev"
246  # The signal to noise limit is good enough, while the flux limit is set
247  # in dimensionless units and may not be appropriate for all data sets.
248  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
249 
250 
251 class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
252  """Assemble DCR coadded images from a set of warps.
253 
254  Attributes
255  ----------
256  bufferSize : `int`
257  The number of pixels to grow each subregion by to allow for DCR.
258 
259  Notes
260  -----
261  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
262  Warps (also called coadded temporary exposures), including the effects of
263  Differential Chromatic Refraction (DCR).
264  For full details of the mathematics and algorithm, please see
265  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
266 
267  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
268  each subfilter used in the iterative calculation.
269  It begins by dividing the bandpass-defining filter into N equal bandwidth
270  "subfilters", and divides the flux in each pixel from an initial coadd
271  equally into each as a "dcrModel". Because the airmass and parallactic
272  angle of each individual exposure is known, we can calculate the shift
273  relative to the center of the band in each subfilter due to DCR. For each
274  exposure we apply this shift as a linear transformation to the dcrModels
275  and stack the results to produce a DCR-matched exposure. The matched
276  exposures are subtracted from the input exposures to produce a set of
277  residual images, and these residuals are reverse shifted for each
278  exposures' subfilters and stacked. The shifted and stacked residuals are
279  added to the dcrModels to produce a new estimate of the flux in each pixel
280  within each subfilter. The dcrModels are solved for iteratively, which
281  continues until the solution from a new iteration improves by less than
282  a set percentage, or a maximum number of iterations is reached.
283  Two forms of regularization are employed to reduce unphysical results.
284  First, the new solution is averaged with the solution from the previous
285  iteration, which mitigates oscillating solutions where the model
286  overshoots with alternating very high and low values.
287  Second, a common degeneracy when the data have a limited range of airmass or
288  parallactic angle values is for one subfilter to be fit with very low or
289  negative values, while another subfilter is fit with very high values. This
290  typically appears in the form of holes next to sources in one subfilter,
291  and corresponding extended wings in another. Because each subfilter has
292  a narrow bandwidth we assume that physical sources that are above the noise
293  level will not vary in flux by more than a factor of `frequencyClampFactor`
294  between subfilters, and pixels that have flux deviations larger than that
295  factor will have the excess flux distributed evenly among all subfilters.
296  If `splitSubfilters` is set, then each subfilter will be further sub-
297  divided during the forward modeling step (only). This approximates using
298  a higher number of subfilters that may be necessary for high airmass
299  observations, but does not increase the number of free parameters in the
300  fit. This is needed when there are high airmass observations which would
301  otherwise have significant DCR even within a subfilter. Because calculating
302  the shifted images takes most of the time, splitting the subfilters is
303  turned off by way of the `splitThreshold` option for low-airmass
304  observations that do not suffer from DCR within a subfilter.
305  """
306 
307  ConfigClass = DcrAssembleCoaddConfig
308  _DefaultName = "dcrAssembleCoadd"
309 
310  def __init__(self, *args, **kwargs):
311  super().__init__(*args, **kwargs)
312  if self.config.doCalculatePsf:
313  self.schema = afwTable.SourceTable.makeMinimalSchema()
314  self.makeSubtask("detectPsfSources", schema=self.schema)
315  self.makeSubtask("measurePsfSources", schema=self.schema)
316  self.makeSubtask("measurePsf", schema=self.schema)
317 
318  @utils.inheritDoc(pipeBase.PipelineTask)
319  def runQuantum(self, butlerQC, inputRefs, outputRefs):
320  # Docstring to be formatted with info from PipelineTask.runQuantum
321  """
322  Notes
323  -----
324  Assemble a coadd from a set of Warps.
325 
326  PipelineTask (Gen3) entry point to Coadd a set of Warps.
327  Analogous to `runDataRef`, it prepares all the data products to be
328  passed to `run`, and processes the results before returning a struct
329  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
330  Therefore, its inputs are accessed subregion by subregion
331  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
332  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
333  correspond to an update in `runDataRef` while both entry points
334  are used.
335  """
336  inputData = butlerQC.get(inputRefs)
337 
338  # Construct skyInfo expected by run
339  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
340  skyMap = inputData["skyMap"]
341  outputDataId = butlerQC.quantum.dataId
342 
343  inputData['skyInfo'] = makeSkyInfo(skyMap,
344  tractId=outputDataId['tract'],
345  patchId=outputDataId['patch'])
346 
347  # Construct list of input Deferred Datasets
348  # These quack a bit like like Gen2 DataRefs
349  warpRefList = inputData['inputWarps']
350  # Perform same middle steps as `runDataRef` does
351  inputs = self.prepareInputs(warpRefList)
352  self.log.info("Found %d %s", len(inputs.tempExpRefList),
353  self.getTempExpDatasetName(self.warpType))
354  if len(inputs.tempExpRefList) == 0:
355  self.log.warn("No coadd temporary exposures found")
356  return
357 
358  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
359  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
360  inputs.weightList, supplementaryData=supplementaryData)
361 
362  inputData.setdefault('brightObjectMask', None)
363  for subfilter in range(self.config.dcrNumSubfilters):
364  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
365  retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
366  self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
367 
368  if self.config.doWrite:
369  butlerQC.put(retStruct, outputRefs)
370  return retStruct
371 
372  @pipeBase.timeMethod
373  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
374  """Assemble a coadd from a set of warps.
375 
376  Coadd a set of Warps. Compute weights to be applied to each Warp and
377  find scalings to match the photometric zeropoint to a reference Warp.
378  Assemble the Warps using run method.
379  Forward model chromatic effects across multiple subfilters,
380  and subtract from the input Warps to build sets of residuals.
381  Use the residuals to construct a new ``DcrModel`` for each subfilter,
382  and iterate until the model converges.
383  Interpolate over NaNs and optionally write the coadd to disk.
384  Return the coadded exposure.
385 
386  Parameters
387  ----------
388  dataRef : `lsst.daf.persistence.ButlerDataRef`
389  Data reference defining the patch for coaddition and the
390  reference Warp
391  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
392  List of data references to warps. Data to be coadded will be
393  selected from this list based on overlap with the patch defined by
394  the data reference.
395 
396  Returns
397  -------
398  results : `lsst.pipe.base.Struct`
399  The Struct contains the following fields:
400 
401  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
402  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
403  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
404  - ``dcrNImages``: `list` of exposure count images for each subfilter
405  """
406  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
407  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
408 
409  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
410  warpRefList=warpRefList)
411  if results is None:
412  skyInfo = self.getSkyInfo(dataRef)
413  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
414  skyInfo.patchInfo.getIndex())
415  return
416  if self.config.doCalculatePsf:
417  self.measureCoaddPsf(results.coaddExposure)
418  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
419  for subfilter in range(self.config.dcrNumSubfilters):
420  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
421  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
422  self.processResults(results.dcrCoadds[subfilter],
423  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
424  if self.config.doWrite:
425  self.log.info("Persisting dcrCoadd")
426  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
427  numSubfilters=self.config.dcrNumSubfilters)
428  if self.config.doNImage and results.dcrNImages is not None:
429  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
430  numSubfilters=self.config.dcrNumSubfilters)
431 
432  return results
433 
434  @utils.inheritDoc(AssembleCoaddTask)
435  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
436  """Load the previously-generated template coadd.
437 
438  This can be removed entirely once we no longer support the Gen 2 butler.
439 
440  Returns
441  -------
442  templateCoadd : `lsst.pipe.base.Struct`
443  Result struct with components:
444 
445  - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
446  """
447  templateCoadd = butlerQC.get(inputRefs.templateExposure)
448 
449  return pipeBase.Struct(templateCoadd=templateCoadd)
450 
451  def measureCoaddPsf(self, coaddExposure):
452  """Detect sources on the coadd exposure and measure the final PSF.
453 
454  Parameters
455  ----------
456  coaddExposure : `lsst.afw.image.Exposure`
457  The final coadded exposure.
458  """
459  table = afwTable.SourceTable.make(self.schema)
460  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
461  coaddSources = detResults.sources
462  self.measurePsfSources.run(
463  measCat=coaddSources,
464  exposure=coaddExposure
465  )
466  # Measure the PSF on the stacked subfilter coadds if possible.
467  # We should already have a decent estimate of the coadd PSF, however,
468  # so in case of any errors simply log them as a warning and use the
469  # default PSF.
470  try:
471  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
472  except Exception as e:
473  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
474  else:
475  coaddExposure.setPsf(psfResults.psf)
476 
477  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
478  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
479 
480  Sets the property ``bufferSize``.
481 
482  Parameters
483  ----------
484  templateCoadd : `lsst.afw.image.ExposureF`
485  The initial coadd exposure before accounting for DCR.
486  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
487  `lsst.daf.persistence.ButlerDataRef`
488  The data references to the input warped exposures.
489  weightList : `list` of `float`
490  The weight to give each input exposure in the coadd
491  Will be modified in place if ``doAirmassWeight`` is set.
492 
493  Returns
494  -------
495  dcrModels : `lsst.pipe.tasks.DcrModel`
496  Best fit model of the true sky after correcting chromatic effects.
497 
498  Raises
499  ------
500  NotImplementedError
501  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
502  """
503  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
504  filterInfo = templateCoadd.getFilter()
505  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
506  raise NotImplementedError("No minimum/maximum wavelength information found"
507  " in the filter definition! Please add lambdaMin and lambdaMax"
508  " to the Mapper class in your obs package.")
509  tempExpName = self.getTempExpDatasetName(self.warpType)
510  dcrShifts = []
511  airmassDict = {}
512  angleDict = {}
513  psfSizeDict = {}
514  for visitNum, warpExpRef in enumerate(warpRefList):
515  if isinstance(warpExpRef, DeferredDatasetHandle):
516  # Gen 3 API
517  visitInfo = warpExpRef.get(component="visitInfo")
518  psf = warpExpRef.get(component="psf")
519  visit = warpExpRef.datasetRefOrType.dataId["visit"]
520  else:
521  # Gen 2 API. Delete this when Gen 2 retired
522  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
523  psf = warpExpRef.get(tempExpName).getPsf()
524  visit = warpExpRef.dataId["visit"]
525  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
526  airmass = visitInfo.getBoresightAirmass()
527  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
528  airmassDict[visit] = airmass
529  angleDict[visit] = parallacticAngle
530  psfSizeDict[visit] = psfSize
531  if self.config.doAirmassWeight:
532  weightList[visitNum] *= airmass
533  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
534  filterInfo, self.config.dcrNumSubfilters))))
535  self.log.info("Selected airmasses:\n%s", airmassDict)
536  self.log.info("Selected parallactic angles:\n%s", angleDict)
537  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
538  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
539  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
540  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
541  self.config.dcrNumSubfilters,
542  filterInfo=filterInfo,
543  psf=psf)
544  return dcrModels
545 
546  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
547  supplementaryData=None):
548  """Assemble the coadd.
549 
550  Requires additional inputs Struct ``supplementaryData`` to contain a
551  ``templateCoadd`` that serves as the model of the static sky.
552 
553  Find artifacts and apply them to the warps' masks creating a list of
554  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
555  Then pass these alternative masks to the base class's assemble method.
556 
557  Divide the ``templateCoadd`` evenly between each subfilter of a
558  ``DcrModel`` as the starting best estimate of the true wavelength-
559  dependent sky. Forward model the ``DcrModel`` using the known
560  chromatic effects in each subfilter and calculate a convergence metric
561  based on how well the modeled template matches the input warps. If
562  the convergence has not yet reached the desired threshold, then shift
563  and stack the residual images to build a new ``DcrModel``. Apply
564  conditioning to prevent oscillating solutions between iterations or
565  between subfilters.
566 
567  Once the ``DcrModel`` reaches convergence or the maximum number of
568  iterations has been reached, fill the metadata for each subfilter
569  image and make them proper ``coaddExposure``s.
570 
571  Parameters
572  ----------
573  skyInfo : `lsst.pipe.base.Struct`
574  Patch geometry information, from getSkyInfo
575  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
576  `lsst.daf.persistence.ButlerDataRef`
577  The data references to the input warped exposures.
578  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
579  The image scalars correct for the zero point of the exposures.
580  weightList : `list` of `float`
581  The weight to give each input exposure in the coadd
582  supplementaryData : `lsst.pipe.base.Struct`
583  Result struct returned by ``makeSupplementaryData`` with components:
584 
585  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
586 
587  Returns
588  -------
589  result : `lsst.pipe.base.Struct`
590  Result struct with components:
591 
592  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
593  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
594  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
595  - ``dcrNImages``: `list` of exposure count images for each subfilter
596  """
597  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
598  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
599  templateCoadd = supplementaryData.templateCoadd
600  baseMask = templateCoadd.mask.clone()
601  # The variance plane is for each subfilter
602  # and should be proportionately lower than the full-band image
603  baseVariance = templateCoadd.variance.clone()
604  baseVariance /= self.config.dcrNumSubfilters
605  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
606  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
607  templateCoadd.setMask(baseMask)
608  badMaskPlanes = self.config.badMaskPlanes[:]
609  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
610  # This is because pixels in observations that are significantly affect by DCR
611  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
612  # but those are necessary to constrain the DCR model.
613  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
614 
615  stats = self.prepareStats(mask=badPixelMask)
616  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
617  if self.config.doNImage:
618  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
619  spanSetMaskList, stats.ctrl)
620  nImage = afwImage.ImageU(skyInfo.bbox)
621  # Note that this nImage will be a factor of dcrNumSubfilters higher than
622  # the nImage returned by assembleCoadd for most pixels. This is because each
623  # subfilter may have a different nImage, and fractional values are not allowed.
624  for dcrNImage in dcrNImages:
625  nImage += dcrNImage
626  else:
627  dcrNImages = None
628 
629  subregionSize = geom.Extent2I(*self.config.subregionSize)
630  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
631  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
632  subIter = 0
633  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
634  modelIter = 0
635  subIter += 1
636  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
637  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
638  dcrBBox = geom.Box2I(subBBox)
639  dcrBBox.grow(self.bufferSize)
640  dcrBBox.clip(dcrModels.bbox)
641  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
642  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
643  imageScalerList, spanSetMaskList)
644  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
645  warpRefList, weightList, stats.ctrl)
646  self.log.info("Initial convergence : %s", convergenceMetric)
647  convergenceList = [convergenceMetric]
648  gainList = []
649  convergenceCheck = 1.
650  refImage = templateCoadd.image
651  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
652  gain = self.calculateGain(convergenceList, gainList)
653  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
654  stats.ctrl, convergenceMetric, gain,
655  modelWeights, refImage, dcrWeights)
656  if self.config.useConvergence:
657  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
658  warpRefList, weightList, stats.ctrl)
659  if convergenceMetric == 0:
660  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
661  "most likely due to there being no valid data in the region.",
662  skyInfo.patchInfo.getIndex(), subIter)
663  break
664  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
665  if (convergenceCheck < 0) & (modelIter > minNumIter):
666  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
667  "iterations or desired convergence improvement of %s."
668  " Divergence: %s",
669  skyInfo.patchInfo.getIndex(), subIter,
670  self.config.convergenceThreshold, convergenceCheck)
671  break
672  convergenceList.append(convergenceMetric)
673  if modelIter > maxNumIter:
674  if self.config.useConvergence:
675  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
676  "before reaching desired convergence improvement of %s."
677  " Final convergence improvement: %s",
678  skyInfo.patchInfo.getIndex(), subIter,
679  self.config.convergenceThreshold, convergenceCheck)
680  break
681 
682  if self.config.useConvergence:
683  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
684  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
685  modelIter += 1
686  else:
687  if self.config.useConvergence:
688  self.log.info("Coadd patch %s subregion %s finished with "
689  "convergence metric %s after %s iterations",
690  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
691  else:
692  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
693  skyInfo.patchInfo.getIndex(), subIter, modelIter)
694  if self.config.useConvergence and convergenceMetric > 0:
695  self.log.info("Final convergence improvement was %.4f%% overall",
696  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
697 
698  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
699  calibration=self.scaleZeroPoint.getPhotoCalib(),
700  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
701  mask=baseMask,
702  variance=baseVariance)
703  coaddExposure = self.stackCoadd(dcrCoadds)
704  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
705  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
706 
707  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
708  """Calculate the number of exposures contributing to each subfilter.
709 
710  Parameters
711  ----------
712  dcrModels : `lsst.pipe.tasks.DcrModel`
713  Best fit model of the true sky after correcting chromatic effects.
714  bbox : `lsst.geom.box.Box2I`
715  Bounding box of the patch to coadd.
716  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
717  `lsst.daf.persistence.ButlerDataRef`
718  The data references to the input warped exposures.
719  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
720  Each element of the `dict` contains the new mask plane name
721  (e.g. "CLIPPED and/or "NO_DATA") as the key,
722  and the list of SpanSets to apply to the mask.
723  statsCtrl : `lsst.afw.math.StatisticsControl`
724  Statistics control object for coadd
725 
726  Returns
727  -------
728  dcrNImages : `list` of `lsst.afw.image.ImageU`
729  List of exposure count images for each subfilter
730  dcrWeights : `list` of `lsst.afw.image.ImageF`
731  Per-pixel weights for each subfilter.
732  Equal to 1/(number of unmasked images contributing to each pixel).
733  """
734  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
735  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
736  tempExpName = self.getTempExpDatasetName(self.warpType)
737  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
738  if isinstance(warpExpRef, DeferredDatasetHandle):
739  # Gen 3 API
740  exposure = warpExpRef.get(parameters={'bbox': bbox})
741  else:
742  # Gen 2 API. Delete this when Gen 2 retired
743  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
744  visitInfo = exposure.getInfo().getVisitInfo()
745  wcs = exposure.getInfo().getWcs()
746  mask = exposure.mask
747  if altMaskSpans is not None:
748  self.applyAltMaskPlanes(mask, altMaskSpans)
749  weightImage = np.zeros_like(exposure.image.array)
750  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
751  # The weights must be shifted in exactly the same way as the residuals,
752  # because they will be used as the denominator in the weighted average of residuals.
753  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
754  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
755  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
756  dcrWeight.array += shiftedWeights
757  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
758  weightsThreshold = 1.
759  goodPix = dcrWeights[0].array > weightsThreshold
760  for weights in dcrWeights[1:]:
761  goodPix = (weights.array > weightsThreshold) & goodPix
762  for subfilter in range(self.config.dcrNumSubfilters):
763  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
764  dcrWeights[subfilter].array[~goodPix] = 0.
765  dcrNImages[subfilter].array[~goodPix] = 0
766  return (dcrNImages, dcrWeights)
767 
768  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
769  statsCtrl, convergenceMetric,
770  gain, modelWeights, refImage, dcrWeights):
771  """Assemble the DCR coadd for a sub-region.
772 
773  Build a DCR-matched template for each input exposure, then shift the
774  residuals according to the DCR in each subfilter.
775  Stack the shifted residuals and apply them as a correction to the
776  solution from the previous iteration.
777  Restrict the new model solutions from varying by more than a factor of
778  `modelClampFactor` from the last solution, and additionally restrict the
779  individual subfilter models from varying by more than a factor of
780  `frequencyClampFactor` from their average.
781  Finally, mitigate potentially oscillating solutions by averaging the new
782  solution with the solution from the previous iteration, weighted by
783  their convergence metric.
784 
785  Parameters
786  ----------
787  dcrModels : `lsst.pipe.tasks.DcrModel`
788  Best fit model of the true sky after correcting chromatic effects.
789  subExposures : `dict` of `lsst.afw.image.ExposureF`
790  The pre-loaded exposures for the current subregion.
791  bbox : `lsst.geom.box.Box2I`
792  Bounding box of the subregion to coadd.
793  dcrBBox : `lsst.geom.box.Box2I`
794  Sub-region of the coadd which includes a buffer to allow for DCR.
795  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
796  `lsst.daf.persistence.ButlerDataRef`
797  The data references to the input warped exposures.
798  statsCtrl : `lsst.afw.math.StatisticsControl`
799  Statistics control object for coadd
800  convergenceMetric : `float`
801  Quality of fit metric for the matched templates of the input images.
802  gain : `float`, optional
803  Relative weight to give the new solution when updating the model.
804  modelWeights : `numpy.ndarray` or `float`
805  A 2D array of weight values that tapers smoothly to zero away from detected sources.
806  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
807  refImage : `lsst.afw.image.Image`
808  A reference image used to supply the default pixel values.
809  dcrWeights : `list` of `lsst.afw.image.Image`
810  Per-pixel weights for each subfilter.
811  Equal to 1/(number of unmasked images contributing to each pixel).
812  """
813  residualGeneratorList = []
814 
815  for warpExpRef in warpRefList:
816  if isinstance(warpExpRef, DeferredDatasetHandle):
817  visit = warpExpRef.datasetRefOrType.dataId["visit"]
818  else:
819  visit = warpExpRef.dataId["visit"]
820  exposure = subExposures[visit]
821  visitInfo = exposure.getInfo().getVisitInfo()
822  wcs = exposure.getInfo().getWcs()
823  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
824  order=self.config.imageInterpOrder,
825  splitSubfilters=self.config.splitSubfilters,
826  splitThreshold=self.config.splitThreshold,
827  amplifyModel=self.config.accelerateModel)
828  residual = exposure.image.array - templateImage.array
829  # Note that the variance plane here is used to store weights, not the actual variance
830  residual *= exposure.variance.array
831  # The residuals are stored as a list of generators.
832  # This allows the residual for a given subfilter and exposure to be created
833  # on the fly, instead of needing to store them all in memory.
834  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
835 
836  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
837  gain=gain,
838  modelWeights=modelWeights,
839  refImage=refImage,
840  dcrWeights=dcrWeights)
841  dcrModels.assign(dcrSubModelOut, bbox)
842 
843  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
844  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
845 
846  Parameters
847  ----------
848  residual : `numpy.ndarray`
849  The residual masked image for one exposure,
850  after subtracting the matched template
851  visitInfo : `lsst.afw.image.VisitInfo`
852  Metadata for the exposure.
853  wcs : `lsst.afw.geom.SkyWcs`
854  Coordinate system definition (wcs) for the exposure.
855  filterInfo : `lsst.afw.image.Filter`
856  The filter definition, set in the current instruments' obs package.
857  Required for any calculation of DCR, including making matched templates.
858 
859  Yields
860  ------
861  residualImage : `numpy.ndarray`
862  The residual image for the next subfilter, shifted for DCR.
863  """
864  # Pre-calculate the spline-filtered residual image, so that step can be
865  # skipped in the shift calculation in `applyDcr`.
866  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
867  # Note that `splitSubfilters` is always turned off in the reverse direction.
868  # This option introduces additional blurring if applied to the residuals.
869  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
870  splitSubfilters=False)
871  for dcr in dcrShift:
872  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
873  doPrefilter=False, order=self.config.imageInterpOrder)
874 
875  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
876  gain, modelWeights, refImage, dcrWeights):
877  """Calculate a new DcrModel from a set of image residuals.
878 
879  Parameters
880  ----------
881  dcrModels : `lsst.pipe.tasks.DcrModel`
882  Current model of the true sky after correcting chromatic effects.
883  residualGeneratorList : `generator` of `numpy.ndarray`
884  The residual image for the next subfilter, shifted for DCR.
885  dcrBBox : `lsst.geom.box.Box2I`
886  Sub-region of the coadd which includes a buffer to allow for DCR.
887  statsCtrl : `lsst.afw.math.StatisticsControl`
888  Statistics control object for coadd
889  gain : `float`
890  Relative weight to give the new solution when updating the model.
891  modelWeights : `numpy.ndarray` or `float`
892  A 2D array of weight values that tapers smoothly to zero away from detected sources.
893  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
894  refImage : `lsst.afw.image.Image`
895  A reference image used to supply the default pixel values.
896  dcrWeights : `list` of `lsst.afw.image.Image`
897  Per-pixel weights for each subfilter.
898  Equal to 1/(number of unmasked images contributing to each pixel).
899 
900  Returns
901  -------
902  dcrModel : `lsst.pipe.tasks.DcrModel`
903  New model of the true sky after correcting chromatic effects.
904  """
905  newModelImages = []
906  for subfilter, model in enumerate(dcrModels):
907  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
908  residual = np.sum(residualsList, axis=0)
909  residual *= dcrWeights[subfilter][dcrBBox].array
910  # `MaskedImage`s only support in-place addition, so rename for readability
911  newModel = model[dcrBBox].clone()
912  newModel.array += residual
913  # Catch any invalid values
914  badPixels = ~np.isfinite(newModel.array)
915  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
916  if self.config.regularizeModelIterations > 0:
917  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
918  self.config.regularizeModelIterations,
919  self.config.regularizationWidth)
920  newModelImages.append(newModel)
921  if self.config.regularizeModelFrequency > 0:
922  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
923  self.config.regularizeModelFrequency,
924  self.config.regularizationWidth)
925  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
926  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
927  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
928  dcrModels.mask, dcrModels.variance)
929 
930  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
931  """Calculate a quality of fit metric for the matched templates.
932 
933  Parameters
934  ----------
935  dcrModels : `lsst.pipe.tasks.DcrModel`
936  Best fit model of the true sky after correcting chromatic effects.
937  subExposures : `dict` of `lsst.afw.image.ExposureF`
938  The pre-loaded exposures for the current subregion.
939  bbox : `lsst.geom.box.Box2I`
940  Sub-region to coadd
941  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
942  `lsst.daf.persistence.ButlerDataRef`
943  The data references to the input warped exposures.
944  weightList : `list` of `float`
945  The weight to give each input exposure in the coadd
946  statsCtrl : `lsst.afw.math.StatisticsControl`
947  Statistics control object for coadd
948 
949  Returns
950  -------
951  convergenceMetric : `float`
952  Quality of fit metric for all input exposures, within the sub-region
953  """
954  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
955  nSigma = 3.
956  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
957  bufferSize=self.bufferSize)
958  if np.max(significanceImage) == 0:
959  significanceImage += 1.
960  weight = 0
961  metric = 0.
962  metricList = {}
963  for warpExpRef, expWeight in zip(warpRefList, weightList):
964  if isinstance(warpExpRef, DeferredDatasetHandle):
965  visit = warpExpRef.datasetRefOrType.dataId["visit"]
966  else:
967  visit = warpExpRef.dataId["visit"]
968  exposure = subExposures[visit][bbox]
969  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
970  metric += singleMetric
971  metricList[visit] = singleMetric
972  weight += 1.
973  self.log.info("Individual metrics:\n%s", metricList)
974  return 1.0 if weight == 0.0 else metric/weight
975 
976  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
977  """Calculate a quality of fit metric for a single matched template.
978 
979  Parameters
980  ----------
981  dcrModels : `lsst.pipe.tasks.DcrModel`
982  Best fit model of the true sky after correcting chromatic effects.
983  exposure : `lsst.afw.image.ExposureF`
984  The input warped exposure to evaluate.
985  significanceImage : `numpy.ndarray`
986  Array of weights for each pixel corresponding to its significance
987  for the convergence calculation.
988  statsCtrl : `lsst.afw.math.StatisticsControl`
989  Statistics control object for coadd
990 
991  Returns
992  -------
993  convergenceMetric : `float`
994  Quality of fit metric for one exposure, within the sub-region.
995  """
996  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
997  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
998  order=self.config.imageInterpOrder,
999  splitSubfilters=self.config.splitSubfilters,
1000  splitThreshold=self.config.splitThreshold,
1001  amplifyModel=self.config.accelerateModel)
1002  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1003  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1004 
1005  finitePixels = np.isfinite(diffVals)
1006  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1007  convergeMaskPixels = exposure.mask.array & convergeMask > 0
1008  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1009  if np.sum(usePixels) == 0:
1010  metric = 0.
1011  else:
1012  diffUse = diffVals[usePixels]
1013  refUse = refVals[usePixels]
1014  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1015  return metric
1016 
1017  def stackCoadd(self, dcrCoadds):
1018  """Add a list of sub-band coadds together.
1019 
1020  Parameters
1021  ----------
1022  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1023  A list of coadd exposures, each exposure containing
1024  the model for one subfilter.
1025 
1026  Returns
1027  -------
1028  coaddExposure : `lsst.afw.image.ExposureF`
1029  A single coadd exposure that is the sum of the sub-bands.
1030  """
1031  coaddExposure = dcrCoadds[0].clone()
1032  for coadd in dcrCoadds[1:]:
1033  coaddExposure.maskedImage += coadd.maskedImage
1034  return coaddExposure
1035 
1036  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1037  mask=None, variance=None):
1038  """Create a list of coadd exposures from a list of masked images.
1039 
1040  Parameters
1041  ----------
1042  dcrModels : `lsst.pipe.tasks.DcrModel`
1043  Best fit model of the true sky after correcting chromatic effects.
1044  skyInfo : `lsst.pipe.base.Struct`
1045  Patch geometry information, from getSkyInfo
1046  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1047  `lsst.daf.persistence.ButlerDataRef`
1048  The data references to the input warped exposures.
1049  weightList : `list` of `float`
1050  The weight to give each input exposure in the coadd
1051  calibration : `lsst.afw.Image.PhotoCalib`, optional
1052  Scale factor to set the photometric calibration of an exposure.
1053  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1054  A record of the observations that are included in the coadd.
1055  mask : `lsst.afw.image.Mask`, optional
1056  Optional mask to override the values in the final coadd.
1057  variance : `lsst.afw.image.Image`, optional
1058  Optional variance plane to override the values in the final coadd.
1059 
1060  Returns
1061  -------
1062  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1063  A list of coadd exposures, each exposure containing
1064  the model for one subfilter.
1065  """
1066  dcrCoadds = []
1067  refModel = dcrModels.getReferenceImage()
1068  for model in dcrModels:
1069  if self.config.accelerateModel > 1:
1070  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1071  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1072  if calibration is not None:
1073  coaddExposure.setPhotoCalib(calibration)
1074  if coaddInputs is not None:
1075  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1076  # Set the metadata for the coadd, including PSF and aperture corrections.
1077  self.assembleMetadata(coaddExposure, warpRefList, weightList)
1078  # Overwrite the PSF
1079  coaddExposure.setPsf(dcrModels.psf)
1080  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1081  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1082  maskedImage.image = model
1083  maskedImage.mask = dcrModels.mask
1084  maskedImage.variance = dcrModels.variance
1085  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1086  if mask is not None:
1087  coaddExposure.setMask(mask)
1088  if variance is not None:
1089  coaddExposure.setVariance(variance)
1090  dcrCoadds.append(coaddExposure)
1091  return dcrCoadds
1092 
1093  def calculateGain(self, convergenceList, gainList):
1094  """Calculate the gain to use for the current iteration.
1095 
1096  After calculating a new DcrModel, each value is averaged with the
1097  value in the corresponding pixel from the previous iteration. This
1098  reduces oscillating solutions that iterative techniques are plagued by,
1099  and speeds convergence. By far the biggest changes to the model
1100  happen in the first couple iterations, so we can also use a more
1101  aggressive gain later when the model is changing slowly.
1102 
1103  Parameters
1104  ----------
1105  convergenceList : `list` of `float`
1106  The quality of fit metric from each previous iteration.
1107  gainList : `list` of `float`
1108  The gains used in each previous iteration: appended with the new
1109  gain value.
1110  Gains are numbers between ``self.config.baseGain`` and 1.
1111 
1112  Returns
1113  -------
1114  gain : `float`
1115  Relative weight to give the new solution when updating the model.
1116  A value of 1.0 gives equal weight to both solutions.
1117 
1118  Raises
1119  ------
1120  ValueError
1121  If ``len(convergenceList) != len(gainList)+1``.
1122  """
1123  nIter = len(convergenceList)
1124  if nIter != len(gainList) + 1:
1125  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1126  % (len(convergenceList), len(gainList)))
1127 
1128  if self.config.baseGain is None:
1129  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1130  # The more subfilters being modeled, the lower the gain should be.
1131  baseGain = 1./(self.config.dcrNumSubfilters - 1)
1132  else:
1133  baseGain = self.config.baseGain
1134 
1135  if self.config.useProgressiveGain and nIter > 2:
1136  # To calculate the best gain to use, compare the past gains that have been used
1137  # with the resulting convergences to estimate the best gain to use.
1138  # Algorithmically, this is a Kalman filter.
1139  # If forward modeling proceeds perfectly, the convergence metric should
1140  # asymptotically approach a final value.
1141  # We can estimate that value from the measured changes in convergence
1142  # weighted by the gains used in each previous iteration.
1143  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1144  for i in range(nIter - 1)]
1145  # The convergence metric is strictly positive, so if the estimated final convergence is
1146  # less than zero, force it to zero.
1147  estFinalConv = np.array(estFinalConv)
1148  estFinalConv[estFinalConv < 0] = 0
1149  # Because the estimate may slowly change over time, only use the most recent measurements.
1150  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1151  lastGain = gainList[-1]
1152  lastConv = convergenceList[-2]
1153  newConv = convergenceList[-1]
1154  # The predicted convergence is the value we would get if the new model calculated
1155  # in the previous iteration was perfect. Recall that the updated model that is
1156  # actually used is the gain-weighted average of the new and old model,
1157  # so the convergence would be similarly weighted.
1158  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1159  # If the measured and predicted convergence are very close, that indicates
1160  # that our forward model is accurate and we can use a more aggressive gain
1161  # If the measured convergence is significantly worse (or better!) than predicted,
1162  # that indicates that the model is not converging as expected and
1163  # we should use a more conservative gain.
1164  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1165  newGain = 1 - abs(delta)
1166  # Average the gains to prevent oscillating solutions.
1167  newGain = (newGain + lastGain)/2.
1168  gain = max(baseGain, newGain)
1169  else:
1170  gain = baseGain
1171  gainList.append(gain)
1172  return gain
1173 
1174  def calculateModelWeights(self, dcrModels, dcrBBox):
1175  """Build an array that smoothly tapers to 0 away from detected sources.
1176 
1177  Parameters
1178  ----------
1179  dcrModels : `lsst.pipe.tasks.DcrModel`
1180  Best fit model of the true sky after correcting chromatic effects.
1181  dcrBBox : `lsst.geom.box.Box2I`
1182  Sub-region of the coadd which includes a buffer to allow for DCR.
1183 
1184  Returns
1185  -------
1186  weights : `numpy.ndarray` or `float`
1187  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1188  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1189 
1190  Raises
1191  ------
1192  ValueError
1193  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1194  """
1195  if not self.config.useModelWeights:
1196  return 1.0
1197  if self.config.modelWeightsWidth < 0:
1198  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1199  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1200  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1201  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1202  weights[convergeMaskPixels] = 1.
1203  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1204  weights /= np.max(weights)
1205  return weights
1206 
1207  def applyModelWeights(self, modelImages, refImage, modelWeights):
1208  """Smoothly replace model pixel values with those from a
1209  reference at locations away from detected sources.
1210 
1211  Parameters
1212  ----------
1213  modelImages : `list` of `lsst.afw.image.Image`
1214  The new DCR model images from the current iteration.
1215  The values will be modified in place.
1216  refImage : `lsst.afw.image.MaskedImage`
1217  A reference image used to supply the default pixel values.
1218  modelWeights : `numpy.ndarray` or `float`
1219  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1220  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1221  """
1222  if self.config.useModelWeights:
1223  for model in modelImages:
1224  model.array *= modelWeights
1225  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1226 
1227  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1228  """Pre-load sub-regions of a list of exposures.
1229 
1230  Parameters
1231  ----------
1232  bbox : `lsst.geom.box.Box2I`
1233  Sub-region to coadd
1234  statsCtrl : `lsst.afw.math.StatisticsControl`
1235  Statistics control object for coadd
1236  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1237  `lsst.daf.persistence.ButlerDataRef`
1238  The data references to the input warped exposures.
1239  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1240  The image scalars correct for the zero point of the exposures.
1241  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1242  Each element is dict with keys = mask plane name to add the spans to
1243 
1244  Returns
1245  -------
1246  subExposures : `dict`
1247  The `dict` keys are the visit IDs,
1248  and the values are `lsst.afw.image.ExposureF`
1249  The pre-loaded exposures for the current subregion.
1250  The variance plane contains weights, and not the variance
1251  """
1252  tempExpName = self.getTempExpDatasetName(self.warpType)
1253  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1254  subExposures = {}
1255  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1256  if isinstance(warpExpRef, DeferredDatasetHandle):
1257  visit = warpExpRef.datasetRefOrType.dataId["visit"]
1258  exposure = warpExpRef.get(parameters={'bbox': bbox})
1259  else:
1260  visit = warpExpRef.dataId["visit"]
1261  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1262  if altMaskSpans is not None:
1263  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1264  imageScaler.scaleMaskedImage(exposure.maskedImage)
1265  # Note that the variance plane here is used to store weights, not the actual variance
1266  exposure.variance.array[:, :] = 0.
1267  # Set the weight of unmasked pixels to 1.
1268  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1269  # Set the image value of masked pixels to zero.
1270  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1271  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1272  subExposures[visit] = exposure
1273  return subExposures
1274 
1275  def selectCoaddPsf(self, templateCoadd, warpRefList):
1276  """Compute the PSF of the coadd from the exposures with the best seeing.
1277 
1278  Parameters
1279  ----------
1280  templateCoadd : `lsst.afw.image.ExposureF`
1281  The initial coadd exposure before accounting for DCR.
1282  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1283  `lsst.daf.persistence.ButlerDataRef`
1284  The data references to the input warped exposures.
1285 
1286  Returns
1287  -------
1288  psf : `lsst.meas.algorithms.CoaddPsf`
1289  The average PSF of the input exposures with the best seeing.
1290  """
1291  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1292  tempExpName = self.getTempExpDatasetName(self.warpType)
1293  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1294  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1295  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1296  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1297  psfSizes = np.zeros(len(ccds))
1298  ccdVisits = np.array(ccds["visit"])
1299  for warpExpRef in warpRefList:
1300  if isinstance(warpExpRef, DeferredDatasetHandle):
1301  # Gen 3 API
1302  psf = warpExpRef.get(component="psf")
1303  visit = warpExpRef.datasetRefOrType.dataId["visit"]
1304  else:
1305  # Gen 2 API. Delete this when Gen 2 retired
1306  psf = warpExpRef.get(tempExpName).getPsf()
1307  visit = warpExpRef.dataId["visit"]
1308  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1309  psfSizes[ccdVisits == visit] = psfSize
1310  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1311  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1312  # of the mean or median FWHM of the input exposures.
1313  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1314  goodPsfs = psfSizes <= sizeThreshold
1315  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1316  self.config.coaddPsf.makeControl())
1317  return psf
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def calculateGain(self, convergenceList, gainList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)