lsst.pipe.tasks  19.0.0-39-g09c7e8d9+4
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32 import lsst.meas.algorithms as measAlg
33 from lsst.meas.base import SingleFrameMeasurementTask
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 import lsst.utils as utils
37 from .assembleCoadd import (AssembleCoaddTask,
38  CompareWarpAssembleCoaddConfig,
39  CompareWarpAssembleCoaddTask)
40 from .coaddBase import makeSkyInfo
41 from .measurePsf import MeasurePsfTask
42 
43 __all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
44 
45 
46 class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47  dimensions=("tract", "patch", "abstract_filter", "skymap"),
48  defaultTemplates={"inputCoaddName": "deep",
49  "outputCoaddName": "dcr",
50  "warpType": "direct",
51  "warpTypeSuffix": "",
52  "fakesType": ""}):
53  inputWarps = pipeBase.connectionTypes.Input(
54  doc=("Input list of warps to be assembled i.e. stacked."
55  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56  name="{inputCoaddName}Coadd_{warpType}Warp",
57  storageClass="ExposureF",
58  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59  deferLoad=True,
60  multiple=True
61  )
62  skyMap = pipeBase.connectionTypes.Input(
63  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64  name="{inputCoaddName}Coadd_skyMap",
65  storageClass="SkyMap",
66  dimensions=("skymap", ),
67  )
68  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70  " BRIGHT_OBJECT."),
71  name="brightObjectMask",
72  storageClass="ObjectMaskCatalog",
73  dimensions=("tract", "patch", "skymap", "abstract_filter"),
74  )
75  templateExposure = pipeBase.connectionTypes.Input(
76  doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77  name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78  storageClass="ExposureF",
79  dimensions=("tract", "patch", "skymap", "abstract_filter"),
80  )
81  dcrCoadds = pipeBase.connectionTypes.Output(
82  doc="Output coadded exposure, produced by stacking input warps",
83  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84  storageClass="ExposureF",
85  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
86  multiple=True,
87  )
88  dcrNImages = pipeBase.connectionTypes.Output(
89  doc="Output image of number of input images per pixel",
90  name="{outputCoaddName}Coadd_nImage",
91  storageClass="ImageU",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  )
95 
96  def __init__(self, *, config=None):
97  super().__init__(config=config)
98  if not config.doWrite:
99  self.outputs.remove("dcrCoadds")
100 
101 
102 class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103  pipelineConnections=DcrAssembleCoaddConnections):
104  dcrNumSubfilters = pexConfig.Field(
105  dtype=int,
106  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107  default=3,
108  )
109  maxNumIter = pexConfig.Field(
110  dtype=int,
111  optional=True,
112  doc="Maximum number of iterations of forward modeling.",
113  default=None,
114  )
115  minNumIter = pexConfig.Field(
116  dtype=int,
117  optional=True,
118  doc="Minimum number of iterations of forward modeling.",
119  default=None,
120  )
121  convergenceThreshold = pexConfig.Field(
122  dtype=float,
123  doc="Target relative change in convergence between iterations of forward modeling.",
124  default=0.001,
125  )
126  useConvergence = pexConfig.Field(
127  dtype=bool,
128  doc="Use convergence test as a forward modeling end condition?"
129  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130  default=True,
131  )
132  baseGain = pexConfig.Field(
133  dtype=float,
134  optional=True,
135  doc="Relative weight to give the new solution vs. the last solution when updating the model."
136  "A value of 1.0 gives equal weight to both solutions."
137  "Small values imply slower convergence of the solution, but can "
138  "help prevent overshooting and failures in the fit."
139  "If ``baseGain`` is None, a conservative gain "
140  "will be calculated from the number of subfilters. ",
141  default=None,
142  )
143  useProgressiveGain = pexConfig.Field(
144  dtype=bool,
145  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146  "When calculating the next gain, we use up to 5 previous gains and convergence values."
147  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148  default=True,
149  )
150  doAirmassWeight = pexConfig.Field(
151  dtype=bool,
152  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153  default=False,
154  )
155  modelWeightsWidth = pexConfig.Field(
156  dtype=float,
157  doc="Width of the region around detected sources to include in the DcrModel.",
158  default=3,
159  )
160  useModelWeights = pexConfig.Field(
161  dtype=bool,
162  doc="Width of the region around detected sources to include in the DcrModel.",
163  default=True,
164  )
165  splitSubfilters = pexConfig.Field(
166  dtype=bool,
167  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168  "Instead of at the midpoint",
169  default=True,
170  )
171  splitThreshold = pexConfig.Field(
172  dtype=float,
173  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174  "Set to 0 to always split the subfilters.",
175  default=0.1,
176  )
177  regularizeModelIterations = pexConfig.Field(
178  dtype=float,
179  doc="Maximum relative change of the model allowed between iterations."
180  "Set to zero to disable.",
181  default=2.,
182  )
183  regularizeModelFrequency = pexConfig.Field(
184  dtype=float,
185  doc="Maximum relative change of the model allowed between subfilters."
186  "Set to zero to disable.",
187  default=4.,
188  )
189  convergenceMaskPlanes = pexConfig.ListField(
190  dtype=str,
191  default=["DETECTED"],
192  doc="Mask planes to use to calculate convergence."
193  )
194  regularizationWidth = pexConfig.Field(
195  dtype=int,
196  default=2,
197  doc="Minimum radius of a region to include in regularization, in pixels."
198  )
199  imageInterpOrder = pexConfig.Field(
200  dtype=int,
201  doc="The order of the spline interpolation used to shift the image plane.",
202  default=3,
203  )
204  accelerateModel = pexConfig.Field(
205  dtype=float,
206  doc="Factor to amplify the differences between model planes by to speed convergence.",
207  default=3,
208  )
209  doCalculatePsf = pexConfig.Field(
210  dtype=bool,
211  doc="Set to detect stars and recalculate the PSF from the final coadd."
212  "Otherwise the PSF is estimated from a selection of the best input exposures",
213  default=False,
214  )
215  detectPsfSources = pexConfig.ConfigurableField(
216  target=measAlg.SourceDetectionTask,
217  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218  )
219  measurePsfSources = pexConfig.ConfigurableField(
220  target=SingleFrameMeasurementTask,
221  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222  )
223  measurePsf = pexConfig.ConfigurableField(
224  target=MeasurePsfTask,
225  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226  )
227 
228  def setDefaults(self):
229  CompareWarpAssembleCoaddConfig.setDefaults(self)
230  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
231  self.doNImage = True
232  self.warpType = "direct"
233  self.assembleStaticSkyModel.warpType = self.warpType
234  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
235  self.assembleStaticSkyModel.doNImage = False
236  self.assembleStaticSkyModel.doWrite = False
237  self.detectPsfSources.returnOriginalFootprints = False
238  self.detectPsfSources.thresholdPolarity = "positive"
239  # Only use bright sources for PSF measurement
240  self.detectPsfSources.thresholdValue = 50
241  self.detectPsfSources.nSigmaToGrow = 2
242  # A valid star for PSF measurement should at least fill 5x5 pixels
243  self.detectPsfSources.minPixels = 25
244  # Use the variance plane to calculate signal to noise
245  self.detectPsfSources.thresholdType = "pixel_stdev"
246  # The signal to noise limit is good enough, while the flux limit is set
247  # in dimensionless units and may not be appropriate for all data sets.
248  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
249 
250 
251 class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
252  """Assemble DCR coadded images from a set of warps.
253 
254  Attributes
255  ----------
256  bufferSize : `int`
257  The number of pixels to grow each subregion by to allow for DCR.
258 
259  Notes
260  -----
261  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
262  Warps (also called coadded temporary exposures), including the effects of
263  Differential Chromatic Refraction (DCR).
264  For full details of the mathematics and algorithm, please see
265  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
266 
267  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
268  each subfilter used in the iterative calculation.
269  It begins by dividing the bandpass-defining filter into N equal bandwidth
270  "subfilters", and divides the flux in each pixel from an initial coadd
271  equally into each as a "dcrModel". Because the airmass and parallactic
272  angle of each individual exposure is known, we can calculate the shift
273  relative to the center of the band in each subfilter due to DCR. For each
274  exposure we apply this shift as a linear transformation to the dcrModels
275  and stack the results to produce a DCR-matched exposure. The matched
276  exposures are subtracted from the input exposures to produce a set of
277  residual images, and these residuals are reverse shifted for each
278  exposures' subfilters and stacked. The shifted and stacked residuals are
279  added to the dcrModels to produce a new estimate of the flux in each pixel
280  within each subfilter. The dcrModels are solved for iteratively, which
281  continues until the solution from a new iteration improves by less than
282  a set percentage, or a maximum number of iterations is reached.
283  Two forms of regularization are employed to reduce unphysical results.
284  First, the new solution is averaged with the solution from the previous
285  iteration, which mitigates oscillating solutions where the model
286  overshoots with alternating very high and low values.
287  Second, a common degeneracy when the data have a limited range of airmass or
288  parallactic angle values is for one subfilter to be fit with very low or
289  negative values, while another subfilter is fit with very high values. This
290  typically appears in the form of holes next to sources in one subfilter,
291  and corresponding extended wings in another. Because each subfilter has
292  a narrow bandwidth we assume that physical sources that are above the noise
293  level will not vary in flux by more than a factor of `frequencyClampFactor`
294  between subfilters, and pixels that have flux deviations larger than that
295  factor will have the excess flux distributed evenly among all subfilters.
296  If `splitSubfilters` is set, then each subfilter will be further sub-
297  divided during the forward modeling step (only). This approximates using
298  a higher number of subfilters that may be necessary for high airmass
299  observations, but does not increase the number of free parameters in the
300  fit. This is needed when there are high airmass observations which would
301  otherwise have significant DCR even within a subfilter. Because calculating
302  the shifted images takes most of the time, splitting the subfilters is
303  turned off by way of the `splitThreshold` option for low-airmass
304  observations that do not suffer from DCR within a subfilter.
305  """
306 
307  ConfigClass = DcrAssembleCoaddConfig
308  _DefaultName = "dcrAssembleCoadd"
309 
310  def __init__(self, *args, **kwargs):
311  super().__init__(*args, **kwargs)
312  if self.config.doCalculatePsf:
313  self.schema = afwTable.SourceTable.makeMinimalSchema()
314  self.makeSubtask("detectPsfSources", schema=self.schema)
315  self.makeSubtask("measurePsfSources", schema=self.schema)
316  self.makeSubtask("measurePsf", schema=self.schema)
317 
318  @utils.inheritDoc(pipeBase.PipelineTask)
319  def runQuantum(self, butlerQC, inputRefs, outputRefs):
320  # Docstring to be formatted with info from PipelineTask.runQuantum
321  """
322  Notes
323  -----
324  Assemble a coadd from a set of Warps.
325 
326  PipelineTask (Gen3) entry point to Coadd a set of Warps.
327  Analogous to `runDataRef`, it prepares all the data products to be
328  passed to `run`, and processes the results before returning a struct
329  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
330  Therefore, its inputs are accessed subregion by subregion
331  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
332  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
333  correspond to an update in `runDataRef` while both entry points
334  are used.
335  """
336  inputData = butlerQC.get(inputRefs)
337 
338  # Construct skyInfo expected by run
339  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
340  skyMap = inputData["skyMap"]
341  outputDataId = butlerQC.quantum.dataId
342 
343  inputData['skyInfo'] = makeSkyInfo(skyMap,
344  tractId=outputDataId['tract'],
345  patchId=outputDataId['patch'])
346 
347  # Construct list of input Deferred Datasets
348  # These quack a bit like like Gen2 DataRefs
349  warpRefList = inputData['inputWarps']
350  # Perform same middle steps as `runDataRef` does
351  inputs = self.prepareInputs(warpRefList)
352  self.log.info("Found %d %s", len(inputs.tempExpRefList),
353  self.getTempExpDatasetName(self.warpType))
354  if len(inputs.tempExpRefList) == 0:
355  self.log.warn("No coadd temporary exposures found")
356  return
357 
358  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
359  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
360  inputs.weightList, supplementaryData=supplementaryData)
361 
362  inputData.setdefault('brightObjectMask', None)
363  for subfilter in range(self.config.dcrNumSubfilters):
364  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
365  retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
366  self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
367 
368  if self.config.doWrite:
369  butlerQC.put(retStruct, outputRefs)
370  return retStruct
371 
372  @pipeBase.timeMethod
373  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
374  """Assemble a coadd from a set of warps.
375 
376  Coadd a set of Warps. Compute weights to be applied to each Warp and
377  find scalings to match the photometric zeropoint to a reference Warp.
378  Assemble the Warps using run method.
379  Forward model chromatic effects across multiple subfilters,
380  and subtract from the input Warps to build sets of residuals.
381  Use the residuals to construct a new ``DcrModel`` for each subfilter,
382  and iterate until the model converges.
383  Interpolate over NaNs and optionally write the coadd to disk.
384  Return the coadded exposure.
385 
386  Parameters
387  ----------
388  dataRef : `lsst.daf.persistence.ButlerDataRef`
389  Data reference defining the patch for coaddition and the
390  reference Warp
391  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
392  List of data references to warps. Data to be coadded will be
393  selected from this list based on overlap with the patch defined by
394  the data reference.
395 
396  Returns
397  -------
398  results : `lsst.pipe.base.Struct`
399  The Struct contains the following fields:
400 
401  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
402  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
403  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
404  - ``dcrNImages``: `list` of exposure count images for each subfilter
405  """
406  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
407  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
408 
409  results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
410  warpRefList=warpRefList)
411  if results is None:
412  skyInfo = self.getSkyInfo(dataRef)
413  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
414  skyInfo.patchInfo.getIndex())
415  return
416  if self.config.doCalculatePsf:
417  self.measureCoaddPsf(results.coaddExposure)
418  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
419  for subfilter in range(self.config.dcrNumSubfilters):
420  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
421  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
422  self.processResults(results.dcrCoadds[subfilter],
423  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
424  if self.config.doWrite:
425  self.log.info("Persisting dcrCoadd")
426  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
427  numSubfilters=self.config.dcrNumSubfilters)
428  if self.config.doNImage and results.dcrNImages is not None:
429  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
430  numSubfilters=self.config.dcrNumSubfilters)
431 
432  return results
433 
434  @utils.inheritDoc(AssembleCoaddTask)
435  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
436  """Load the previously-generated template coadd.
437 
438  This can be removed entirely once we no longer support the Gen 2 butler.
439 
440  Returns
441  -------
442  templateCoadd : `lsst.pipe.base.Struct`
443  Result struct with components:
444 
445  - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
446  """
447  templateCoadd = butlerQC.get(inputRefs.templateExposure)
448 
449  return pipeBase.Struct(templateCoadd=templateCoadd)
450 
451  def measureCoaddPsf(self, coaddExposure):
452  """Detect sources on the coadd exposure and measure the final PSF.
453 
454  Parameters
455  ----------
456  coaddExposure : `lsst.afw.image.Exposure`
457  The final coadded exposure.
458  """
459  table = afwTable.SourceTable.make(self.schema)
460  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
461  coaddSources = detResults.sources
462  self.measurePsfSources.run(
463  measCat=coaddSources,
464  exposure=coaddExposure
465  )
466  # Measure the PSF on the stacked subfilter coadds if possible.
467  # We should already have a decent estimate of the coadd PSF, however,
468  # so in case of any errors simply log them as a warning and use the
469  # default PSF.
470  try:
471  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
472  except Exception as e:
473  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
474  else:
475  coaddExposure.setPsf(psfResults.psf)
476 
477  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
478  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
479 
480  Sets the property ``bufferSize``.
481 
482  Parameters
483  ----------
484  templateCoadd : `lsst.afw.image.ExposureF`
485  The initial coadd exposure before accounting for DCR.
486  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
487  `lsst.daf.persistence.ButlerDataRef`
488  The data references to the input warped exposures.
489  weightList : `list` of `float`
490  The weight to give each input exposure in the coadd
491  Will be modified in place if ``doAirmassWeight`` is set.
492 
493  Returns
494  -------
495  dcrModels : `lsst.pipe.tasks.DcrModel`
496  Best fit model of the true sky after correcting chromatic effects.
497 
498  Raises
499  ------
500  NotImplementedError
501  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
502  """
503  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
504  filterInfo = templateCoadd.getFilter()
505  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
506  raise NotImplementedError("No minimum/maximum wavelength information found"
507  " in the filter definition! Please add lambdaMin and lambdaMax"
508  " to the Mapper class in your obs package.")
509  tempExpName = self.getTempExpDatasetName(self.warpType)
510  dcrShifts = []
511  airmassDict = {}
512  angleDict = {}
513  psfSizeDict = {}
514  for visitNum, warpExpRef in enumerate(warpRefList):
515  if isinstance(warpExpRef, DeferredDatasetHandle):
516  # Gen 3 API
517  visitInfo = warpExpRef.get(component="visitInfo")
518  psf = warpExpRef.get(component="psf")
519  else:
520  # Gen 2 API. Delete this when Gen 2 retired
521  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
522  psf = warpExpRef.get(tempExpName).getPsf()
523  visit = warpExpRef.dataId["visit"]
524  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
525  airmass = visitInfo.getBoresightAirmass()
526  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
527  airmassDict[visit] = airmass
528  angleDict[visit] = parallacticAngle
529  psfSizeDict[visit] = psfSize
530  if self.config.doAirmassWeight:
531  weightList[visitNum] *= airmass
532  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
533  filterInfo, self.config.dcrNumSubfilters))))
534  self.log.info("Selected airmasses:\n%s", airmassDict)
535  self.log.info("Selected parallactic angles:\n%s", angleDict)
536  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
537  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
538  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
539  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
540  self.config.dcrNumSubfilters,
541  filterInfo=filterInfo,
542  psf=psf)
543  return dcrModels
544 
545  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
546  supplementaryData=None):
547  """Assemble the coadd.
548 
549  Requires additional inputs Struct ``supplementaryData`` to contain a
550  ``templateCoadd`` that serves as the model of the static sky.
551 
552  Find artifacts and apply them to the warps' masks creating a list of
553  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
554  Then pass these alternative masks to the base class's assemble method.
555 
556  Divide the ``templateCoadd`` evenly between each subfilter of a
557  ``DcrModel`` as the starting best estimate of the true wavelength-
558  dependent sky. Forward model the ``DcrModel`` using the known
559  chromatic effects in each subfilter and calculate a convergence metric
560  based on how well the modeled template matches the input warps. If
561  the convergence has not yet reached the desired threshold, then shift
562  and stack the residual images to build a new ``DcrModel``. Apply
563  conditioning to prevent oscillating solutions between iterations or
564  between subfilters.
565 
566  Once the ``DcrModel`` reaches convergence or the maximum number of
567  iterations has been reached, fill the metadata for each subfilter
568  image and make them proper ``coaddExposure``s.
569 
570  Parameters
571  ----------
572  skyInfo : `lsst.pipe.base.Struct`
573  Patch geometry information, from getSkyInfo
574  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
575  `lsst.daf.persistence.ButlerDataRef`
576  The data references to the input warped exposures.
577  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
578  The image scalars correct for the zero point of the exposures.
579  weightList : `list` of `float`
580  The weight to give each input exposure in the coadd
581  supplementaryData : `lsst.pipe.base.Struct`
582  Result struct returned by ``makeSupplementaryData`` with components:
583 
584  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
585 
586  Returns
587  -------
588  result : `lsst.pipe.base.Struct`
589  Result struct with components:
590 
591  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
592  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
593  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
594  - ``dcrNImages``: `list` of exposure count images for each subfilter
595  """
596  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
597  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
598  templateCoadd = supplementaryData.templateCoadd
599  baseMask = templateCoadd.mask.clone()
600  # The variance plane is for each subfilter
601  # and should be proportionately lower than the full-band image
602  baseVariance = templateCoadd.variance.clone()
603  baseVariance /= self.config.dcrNumSubfilters
604  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
605  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
606  templateCoadd.setMask(baseMask)
607  badMaskPlanes = self.config.badMaskPlanes[:]
608  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
609  # This is because pixels in observations that are significantly affect by DCR
610  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
611  # but those are necessary to constrain the DCR model.
612  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
613 
614  stats = self.prepareStats(mask=badPixelMask)
615  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
616  if self.config.doNImage:
617  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
618  spanSetMaskList, stats.ctrl)
619  nImage = afwImage.ImageU(skyInfo.bbox)
620  # Note that this nImage will be a factor of dcrNumSubfilters higher than
621  # the nImage returned by assembleCoadd for most pixels. This is because each
622  # subfilter may have a different nImage, and fractional values are not allowed.
623  for dcrNImage in dcrNImages:
624  nImage += dcrNImage
625  else:
626  dcrNImages = None
627 
628  subregionSize = geom.Extent2I(*self.config.subregionSize)
629  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
630  ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
631  subIter = 0
632  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
633  modelIter = 0
634  subIter += 1
635  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
636  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
637  dcrBBox = geom.Box2I(subBBox)
638  dcrBBox.grow(self.bufferSize)
639  dcrBBox.clip(dcrModels.bbox)
640  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
641  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
642  imageScalerList, spanSetMaskList)
643  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
644  warpRefList, weightList, stats.ctrl)
645  self.log.info("Initial convergence : %s", convergenceMetric)
646  convergenceList = [convergenceMetric]
647  gainList = []
648  convergenceCheck = 1.
649  refImage = templateCoadd.image
650  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
651  gain = self.calculateGain(convergenceList, gainList)
652  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
653  stats.ctrl, convergenceMetric, gain,
654  modelWeights, refImage, dcrWeights)
655  if self.config.useConvergence:
656  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
657  warpRefList, weightList, stats.ctrl)
658  if convergenceMetric == 0:
659  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
660  "most likely due to there being no valid data in the region.",
661  skyInfo.patchInfo.getIndex(), subIter)
662  break
663  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
664  if (convergenceCheck < 0) & (modelIter > minNumIter):
665  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
666  "iterations or desired convergence improvement of %s."
667  " Divergence: %s",
668  skyInfo.patchInfo.getIndex(), subIter,
669  self.config.convergenceThreshold, convergenceCheck)
670  break
671  convergenceList.append(convergenceMetric)
672  if modelIter > maxNumIter:
673  if self.config.useConvergence:
674  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
675  "before reaching desired convergence improvement of %s."
676  " Final convergence improvement: %s",
677  skyInfo.patchInfo.getIndex(), subIter,
678  self.config.convergenceThreshold, convergenceCheck)
679  break
680 
681  if self.config.useConvergence:
682  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
683  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
684  modelIter += 1
685  else:
686  if self.config.useConvergence:
687  self.log.info("Coadd patch %s subregion %s finished with "
688  "convergence metric %s after %s iterations",
689  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
690  else:
691  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
692  skyInfo.patchInfo.getIndex(), subIter, modelIter)
693  if self.config.useConvergence and convergenceMetric > 0:
694  self.log.info("Final convergence improvement was %.4f%% overall",
695  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
696 
697  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
698  calibration=self.scaleZeroPoint.getPhotoCalib(),
699  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
700  mask=baseMask,
701  variance=baseVariance)
702  coaddExposure = self.stackCoadd(dcrCoadds)
703  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
704  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
705 
706  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
707  """Calculate the number of exposures contributing to each subfilter.
708 
709  Parameters
710  ----------
711  dcrModels : `lsst.pipe.tasks.DcrModel`
712  Best fit model of the true sky after correcting chromatic effects.
713  bbox : `lsst.geom.box.Box2I`
714  Bounding box of the patch to coadd.
715  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
716  `lsst.daf.persistence.ButlerDataRef`
717  The data references to the input warped exposures.
718  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
719  Each element of the `dict` contains the new mask plane name
720  (e.g. "CLIPPED and/or "NO_DATA") as the key,
721  and the list of SpanSets to apply to the mask.
722  statsCtrl : `lsst.afw.math.StatisticsControl`
723  Statistics control object for coadd
724 
725  Returns
726  -------
727  dcrNImages : `list` of `lsst.afw.image.ImageU`
728  List of exposure count images for each subfilter
729  dcrWeights : `list` of `lsst.afw.image.ImageF`
730  Per-pixel weights for each subfilter.
731  Equal to 1/(number of unmasked images contributing to each pixel).
732  """
733  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
734  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
735  tempExpName = self.getTempExpDatasetName(self.warpType)
736  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
737  if isinstance(warpExpRef, DeferredDatasetHandle):
738  # Gen 3 API
739  exposure = warpExpRef.get(parameters={'bbox': bbox})
740  else:
741  # Gen 2 API. Delete this when Gen 2 retired
742  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
743  visitInfo = exposure.getInfo().getVisitInfo()
744  wcs = exposure.getInfo().getWcs()
745  mask = exposure.mask
746  if altMaskSpans is not None:
747  self.applyAltMaskPlanes(mask, altMaskSpans)
748  weightImage = np.zeros_like(exposure.image.array)
749  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
750  # The weights must be shifted in exactly the same way as the residuals,
751  # because they will be used as the denominator in the weighted average of residuals.
752  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
753  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
754  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
755  dcrWeight.array += shiftedWeights
756  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
757  weightsThreshold = 1.
758  goodPix = dcrWeights[0].array > weightsThreshold
759  for weights in dcrWeights[1:]:
760  goodPix = (weights.array > weightsThreshold) & goodPix
761  for subfilter in range(self.config.dcrNumSubfilters):
762  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
763  dcrWeights[subfilter].array[~goodPix] = 0.
764  dcrNImages[subfilter].array[~goodPix] = 0
765  return (dcrNImages, dcrWeights)
766 
767  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
768  statsCtrl, convergenceMetric,
769  gain, modelWeights, refImage, dcrWeights):
770  """Assemble the DCR coadd for a sub-region.
771 
772  Build a DCR-matched template for each input exposure, then shift the
773  residuals according to the DCR in each subfilter.
774  Stack the shifted residuals and apply them as a correction to the
775  solution from the previous iteration.
776  Restrict the new model solutions from varying by more than a factor of
777  `modelClampFactor` from the last solution, and additionally restrict the
778  individual subfilter models from varying by more than a factor of
779  `frequencyClampFactor` from their average.
780  Finally, mitigate potentially oscillating solutions by averaging the new
781  solution with the solution from the previous iteration, weighted by
782  their convergence metric.
783 
784  Parameters
785  ----------
786  dcrModels : `lsst.pipe.tasks.DcrModel`
787  Best fit model of the true sky after correcting chromatic effects.
788  subExposures : `dict` of `lsst.afw.image.ExposureF`
789  The pre-loaded exposures for the current subregion.
790  bbox : `lsst.geom.box.Box2I`
791  Bounding box of the subregion to coadd.
792  dcrBBox : `lsst.geom.box.Box2I`
793  Sub-region of the coadd which includes a buffer to allow for DCR.
794  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
795  `lsst.daf.persistence.ButlerDataRef`
796  The data references to the input warped exposures.
797  statsCtrl : `lsst.afw.math.StatisticsControl`
798  Statistics control object for coadd
799  convergenceMetric : `float`
800  Quality of fit metric for the matched templates of the input images.
801  gain : `float`, optional
802  Relative weight to give the new solution when updating the model.
803  modelWeights : `numpy.ndarray` or `float`
804  A 2D array of weight values that tapers smoothly to zero away from detected sources.
805  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
806  refImage : `lsst.afw.image.Image`
807  A reference image used to supply the default pixel values.
808  dcrWeights : `list` of `lsst.afw.image.Image`
809  Per-pixel weights for each subfilter.
810  Equal to 1/(number of unmasked images contributing to each pixel).
811  """
812  residualGeneratorList = []
813 
814  for warpExpRef in warpRefList:
815  visit = warpExpRef.dataId["visit"]
816  exposure = subExposures[visit]
817  visitInfo = exposure.getInfo().getVisitInfo()
818  wcs = exposure.getInfo().getWcs()
819  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
820  order=self.config.imageInterpOrder,
821  splitSubfilters=self.config.splitSubfilters,
822  splitThreshold=self.config.splitThreshold,
823  amplifyModel=self.config.accelerateModel)
824  residual = exposure.image.array - templateImage.array
825  # Note that the variance plane here is used to store weights, not the actual variance
826  residual *= exposure.variance.array
827  # The residuals are stored as a list of generators.
828  # This allows the residual for a given subfilter and exposure to be created
829  # on the fly, instead of needing to store them all in memory.
830  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
831 
832  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
833  gain=gain,
834  modelWeights=modelWeights,
835  refImage=refImage,
836  dcrWeights=dcrWeights)
837  dcrModels.assign(dcrSubModelOut, bbox)
838 
839  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
840  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
841 
842  Parameters
843  ----------
844  residual : `numpy.ndarray`
845  The residual masked image for one exposure,
846  after subtracting the matched template
847  visitInfo : `lsst.afw.image.VisitInfo`
848  Metadata for the exposure.
849  wcs : `lsst.afw.geom.SkyWcs`
850  Coordinate system definition (wcs) for the exposure.
851  filterInfo : `lsst.afw.image.Filter`
852  The filter definition, set in the current instruments' obs package.
853  Required for any calculation of DCR, including making matched templates.
854 
855  Yields
856  ------
857  residualImage : `numpy.ndarray`
858  The residual image for the next subfilter, shifted for DCR.
859  """
860  # Pre-calculate the spline-filtered residual image, so that step can be
861  # skipped in the shift calculation in `applyDcr`.
862  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
863  # Note that `splitSubfilters` is always turned off in the reverse direction.
864  # This option introduces additional blurring if applied to the residuals.
865  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
866  splitSubfilters=False)
867  for dcr in dcrShift:
868  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
869  doPrefilter=False, order=self.config.imageInterpOrder)
870 
871  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
872  gain, modelWeights, refImage, dcrWeights):
873  """Calculate a new DcrModel from a set of image residuals.
874 
875  Parameters
876  ----------
877  dcrModels : `lsst.pipe.tasks.DcrModel`
878  Current model of the true sky after correcting chromatic effects.
879  residualGeneratorList : `generator` of `numpy.ndarray`
880  The residual image for the next subfilter, shifted for DCR.
881  dcrBBox : `lsst.geom.box.Box2I`
882  Sub-region of the coadd which includes a buffer to allow for DCR.
883  statsCtrl : `lsst.afw.math.StatisticsControl`
884  Statistics control object for coadd
885  gain : `float`
886  Relative weight to give the new solution when updating the model.
887  modelWeights : `numpy.ndarray` or `float`
888  A 2D array of weight values that tapers smoothly to zero away from detected sources.
889  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
890  refImage : `lsst.afw.image.Image`
891  A reference image used to supply the default pixel values.
892  dcrWeights : `list` of `lsst.afw.image.Image`
893  Per-pixel weights for each subfilter.
894  Equal to 1/(number of unmasked images contributing to each pixel).
895 
896  Returns
897  -------
898  dcrModel : `lsst.pipe.tasks.DcrModel`
899  New model of the true sky after correcting chromatic effects.
900  """
901  newModelImages = []
902  for subfilter, model in enumerate(dcrModels):
903  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
904  residual = np.sum(residualsList, axis=0)
905  residual *= dcrWeights[subfilter][dcrBBox].array
906  # `MaskedImage`s only support in-place addition, so rename for readability
907  newModel = model[dcrBBox].clone()
908  newModel.array += residual
909  # Catch any invalid values
910  badPixels = ~np.isfinite(newModel.array)
911  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
912  if self.config.regularizeModelIterations > 0:
913  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
914  self.config.regularizeModelIterations,
915  self.config.regularizationWidth)
916  newModelImages.append(newModel)
917  if self.config.regularizeModelFrequency > 0:
918  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
919  self.config.regularizeModelFrequency,
920  self.config.regularizationWidth)
921  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
922  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
923  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
924  dcrModels.mask, dcrModels.variance)
925 
926  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
927  """Calculate a quality of fit metric for the matched templates.
928 
929  Parameters
930  ----------
931  dcrModels : `lsst.pipe.tasks.DcrModel`
932  Best fit model of the true sky after correcting chromatic effects.
933  subExposures : `dict` of `lsst.afw.image.ExposureF`
934  The pre-loaded exposures for the current subregion.
935  bbox : `lsst.geom.box.Box2I`
936  Sub-region to coadd
937  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
938  `lsst.daf.persistence.ButlerDataRef`
939  The data references to the input warped exposures.
940  weightList : `list` of `float`
941  The weight to give each input exposure in the coadd
942  statsCtrl : `lsst.afw.math.StatisticsControl`
943  Statistics control object for coadd
944 
945  Returns
946  -------
947  convergenceMetric : `float`
948  Quality of fit metric for all input exposures, within the sub-region
949  """
950  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
951  nSigma = 3.
952  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
953  bufferSize=self.bufferSize)
954  if np.max(significanceImage) == 0:
955  significanceImage += 1.
956  weight = 0
957  metric = 0.
958  metricList = {}
959  for warpExpRef, expWeight in zip(warpRefList, weightList):
960  visit = warpExpRef.dataId["visit"]
961  exposure = subExposures[visit][bbox]
962  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
963  metric += singleMetric
964  metricList[visit] = singleMetric
965  weight += 1.
966  self.log.info("Individual metrics:\n%s", metricList)
967  return 1.0 if weight == 0.0 else metric/weight
968 
969  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
970  """Calculate a quality of fit metric for a single matched template.
971 
972  Parameters
973  ----------
974  dcrModels : `lsst.pipe.tasks.DcrModel`
975  Best fit model of the true sky after correcting chromatic effects.
976  exposure : `lsst.afw.image.ExposureF`
977  The input warped exposure to evaluate.
978  significanceImage : `numpy.ndarray`
979  Array of weights for each pixel corresponding to its significance
980  for the convergence calculation.
981  statsCtrl : `lsst.afw.math.StatisticsControl`
982  Statistics control object for coadd
983 
984  Returns
985  -------
986  convergenceMetric : `float`
987  Quality of fit metric for one exposure, within the sub-region.
988  """
989  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
990  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
991  order=self.config.imageInterpOrder,
992  splitSubfilters=self.config.splitSubfilters,
993  splitThreshold=self.config.splitThreshold,
994  amplifyModel=self.config.accelerateModel)
995  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
996  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
997 
998  finitePixels = np.isfinite(diffVals)
999  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1000  convergeMaskPixels = exposure.mask.array & convergeMask > 0
1001  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1002  if np.sum(usePixels) == 0:
1003  metric = 0.
1004  else:
1005  diffUse = diffVals[usePixels]
1006  refUse = refVals[usePixels]
1007  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1008  return metric
1009 
1010  def stackCoadd(self, dcrCoadds):
1011  """Add a list of sub-band coadds together.
1012 
1013  Parameters
1014  ----------
1015  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1016  A list of coadd exposures, each exposure containing
1017  the model for one subfilter.
1018 
1019  Returns
1020  -------
1021  coaddExposure : `lsst.afw.image.ExposureF`
1022  A single coadd exposure that is the sum of the sub-bands.
1023  """
1024  coaddExposure = dcrCoadds[0].clone()
1025  for coadd in dcrCoadds[1:]:
1026  coaddExposure.maskedImage += coadd.maskedImage
1027  return coaddExposure
1028 
1029  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1030  mask=None, variance=None):
1031  """Create a list of coadd exposures from a list of masked images.
1032 
1033  Parameters
1034  ----------
1035  dcrModels : `lsst.pipe.tasks.DcrModel`
1036  Best fit model of the true sky after correcting chromatic effects.
1037  skyInfo : `lsst.pipe.base.Struct`
1038  Patch geometry information, from getSkyInfo
1039  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1040  `lsst.daf.persistence.ButlerDataRef`
1041  The data references to the input warped exposures.
1042  weightList : `list` of `float`
1043  The weight to give each input exposure in the coadd
1044  calibration : `lsst.afw.Image.PhotoCalib`, optional
1045  Scale factor to set the photometric calibration of an exposure.
1046  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1047  A record of the observations that are included in the coadd.
1048  mask : `lsst.afw.image.Mask`, optional
1049  Optional mask to override the values in the final coadd.
1050  variance : `lsst.afw.image.Image`, optional
1051  Optional variance plane to override the values in the final coadd.
1052 
1053  Returns
1054  -------
1055  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1056  A list of coadd exposures, each exposure containing
1057  the model for one subfilter.
1058  """
1059  dcrCoadds = []
1060  refModel = dcrModels.getReferenceImage()
1061  for model in dcrModels:
1062  if self.config.accelerateModel > 1:
1063  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1064  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1065  if calibration is not None:
1066  coaddExposure.setPhotoCalib(calibration)
1067  if coaddInputs is not None:
1068  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1069  # Set the metadata for the coadd, including PSF and aperture corrections.
1070  self.assembleMetadata(coaddExposure, warpRefList, weightList)
1071  # Overwrite the PSF
1072  coaddExposure.setPsf(dcrModels.psf)
1073  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1074  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1075  maskedImage.image = model
1076  maskedImage.mask = dcrModels.mask
1077  maskedImage.variance = dcrModels.variance
1078  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1079  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1080  if mask is not None:
1081  coaddExposure.setMask(mask)
1082  if variance is not None:
1083  coaddExposure.setVariance(variance)
1084  dcrCoadds.append(coaddExposure)
1085  return dcrCoadds
1086 
1087  def calculateGain(self, convergenceList, gainList):
1088  """Calculate the gain to use for the current iteration.
1089 
1090  After calculating a new DcrModel, each value is averaged with the
1091  value in the corresponding pixel from the previous iteration. This
1092  reduces oscillating solutions that iterative techniques are plagued by,
1093  and speeds convergence. By far the biggest changes to the model
1094  happen in the first couple iterations, so we can also use a more
1095  aggressive gain later when the model is changing slowly.
1096 
1097  Parameters
1098  ----------
1099  convergenceList : `list` of `float`
1100  The quality of fit metric from each previous iteration.
1101  gainList : `list` of `float`
1102  The gains used in each previous iteration: appended with the new
1103  gain value.
1104  Gains are numbers between ``self.config.baseGain`` and 1.
1105 
1106  Returns
1107  -------
1108  gain : `float`
1109  Relative weight to give the new solution when updating the model.
1110  A value of 1.0 gives equal weight to both solutions.
1111 
1112  Raises
1113  ------
1114  ValueError
1115  If ``len(convergenceList) != len(gainList)+1``.
1116  """
1117  nIter = len(convergenceList)
1118  if nIter != len(gainList) + 1:
1119  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1120  % (len(convergenceList), len(gainList)))
1121 
1122  if self.config.baseGain is None:
1123  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1124  # The more subfilters being modeled, the lower the gain should be.
1125  baseGain = 1./(self.config.dcrNumSubfilters - 1)
1126  else:
1127  baseGain = self.config.baseGain
1128 
1129  if self.config.useProgressiveGain and nIter > 2:
1130  # To calculate the best gain to use, compare the past gains that have been used
1131  # with the resulting convergences to estimate the best gain to use.
1132  # Algorithmically, this is a Kalman filter.
1133  # If forward modeling proceeds perfectly, the convergence metric should
1134  # asymptotically approach a final value.
1135  # We can estimate that value from the measured changes in convergence
1136  # weighted by the gains used in each previous iteration.
1137  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1138  for i in range(nIter - 1)]
1139  # The convergence metric is strictly positive, so if the estimated final convergence is
1140  # less than zero, force it to zero.
1141  estFinalConv = np.array(estFinalConv)
1142  estFinalConv[estFinalConv < 0] = 0
1143  # Because the estimate may slowly change over time, only use the most recent measurements.
1144  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1145  lastGain = gainList[-1]
1146  lastConv = convergenceList[-2]
1147  newConv = convergenceList[-1]
1148  # The predicted convergence is the value we would get if the new model calculated
1149  # in the previous iteration was perfect. Recall that the updated model that is
1150  # actually used is the gain-weighted average of the new and old model,
1151  # so the convergence would be similarly weighted.
1152  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1153  # If the measured and predicted convergence are very close, that indicates
1154  # that our forward model is accurate and we can use a more aggressive gain
1155  # If the measured convergence is significantly worse (or better!) than predicted,
1156  # that indicates that the model is not converging as expected and
1157  # we should use a more conservative gain.
1158  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1159  newGain = 1 - abs(delta)
1160  # Average the gains to prevent oscillating solutions.
1161  newGain = (newGain + lastGain)/2.
1162  gain = max(baseGain, newGain)
1163  else:
1164  gain = baseGain
1165  gainList.append(gain)
1166  return gain
1167 
1168  def calculateModelWeights(self, dcrModels, dcrBBox):
1169  """Build an array that smoothly tapers to 0 away from detected sources.
1170 
1171  Parameters
1172  ----------
1173  dcrModels : `lsst.pipe.tasks.DcrModel`
1174  Best fit model of the true sky after correcting chromatic effects.
1175  dcrBBox : `lsst.geom.box.Box2I`
1176  Sub-region of the coadd which includes a buffer to allow for DCR.
1177 
1178  Returns
1179  -------
1180  weights : `numpy.ndarray` or `float`
1181  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1182  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1183 
1184  Raises
1185  ------
1186  ValueError
1187  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1188  """
1189  if not self.config.useModelWeights:
1190  return 1.0
1191  if self.config.modelWeightsWidth < 0:
1192  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1193  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1194  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1195  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1196  weights[convergeMaskPixels] = 1.
1197  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1198  weights /= np.max(weights)
1199  return weights
1200 
1201  def applyModelWeights(self, modelImages, refImage, modelWeights):
1202  """Smoothly replace model pixel values with those from a
1203  reference at locations away from detected sources.
1204 
1205  Parameters
1206  ----------
1207  modelImages : `list` of `lsst.afw.image.Image`
1208  The new DCR model images from the current iteration.
1209  The values will be modified in place.
1210  refImage : `lsst.afw.image.MaskedImage`
1211  A reference image used to supply the default pixel values.
1212  modelWeights : `numpy.ndarray` or `float`
1213  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1214  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1215  """
1216  if self.config.useModelWeights:
1217  for model in modelImages:
1218  model.array *= modelWeights
1219  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1220 
1221  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1222  """Pre-load sub-regions of a list of exposures.
1223 
1224  Parameters
1225  ----------
1226  bbox : `lsst.geom.box.Box2I`
1227  Sub-region to coadd
1228  statsCtrl : `lsst.afw.math.StatisticsControl`
1229  Statistics control object for coadd
1230  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1231  `lsst.daf.persistence.ButlerDataRef`
1232  The data references to the input warped exposures.
1233  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1234  The image scalars correct for the zero point of the exposures.
1235  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1236  Each element is dict with keys = mask plane name to add the spans to
1237 
1238  Returns
1239  -------
1240  subExposures : `dict`
1241  The `dict` keys are the visit IDs,
1242  and the values are `lsst.afw.image.ExposureF`
1243  The pre-loaded exposures for the current subregion.
1244  The variance plane contains weights, and not the variance
1245  """
1246  tempExpName = self.getTempExpDatasetName(self.warpType)
1247  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1248  subExposures = {}
1249  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1250  if isinstance(warpExpRef, DeferredDatasetHandle):
1251  exposure = warpExpRef.get(parameters={'bbox': bbox})
1252  else:
1253  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1254  visit = warpExpRef.dataId["visit"]
1255  if altMaskSpans is not None:
1256  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1257  imageScaler.scaleMaskedImage(exposure.maskedImage)
1258  # Note that the variance plane here is used to store weights, not the actual variance
1259  exposure.variance.array[:, :] = 0.
1260  # Set the weight of unmasked pixels to 1.
1261  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1262  # Set the image value of masked pixels to zero.
1263  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1264  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1265  subExposures[visit] = exposure
1266  return subExposures
1267 
1268  def selectCoaddPsf(self, templateCoadd, warpRefList):
1269  """Compute the PSF of the coadd from the exposures with the best seeing.
1270 
1271  Parameters
1272  ----------
1273  templateCoadd : `lsst.afw.image.ExposureF`
1274  The initial coadd exposure before accounting for DCR.
1275  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1276  `lsst.daf.persistence.ButlerDataRef`
1277  The data references to the input warped exposures.
1278 
1279  Returns
1280  -------
1281  psf : `lsst.meas.algorithms.CoaddPsf`
1282  The average PSF of the input exposures with the best seeing.
1283  """
1284  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1285  tempExpName = self.getTempExpDatasetName(self.warpType)
1286  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1287  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1288  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1289  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1290  psfSizes = np.zeros(len(ccds))
1291  ccdVisits = np.array(ccds["visit"])
1292  for warpExpRef in warpRefList:
1293  if isinstance(warpExpRef, DeferredDatasetHandle):
1294  # Gen 3 API
1295  psf = warpExpRef.get(component="psf")
1296  else:
1297  # Gen 2 API. Delete this when Gen 2 retired
1298  psf = warpExpRef.get(tempExpName).getPsf()
1299  visit = warpExpRef.dataId["visit"]
1300  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1301  psfSizes[ccdVisits == visit] = psfSize
1302  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1303  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1304  # of the mean or median FWHM of the input exposures.
1305  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1306  goodPsfs = psfSizes <= sizeThreshold
1307  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1308  self.config.coaddPsf.makeControl())
1309  return psf
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def calculateGain(self, convergenceList, gainList)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)