lsst.pipe.tasks  20.0.0-21-g7fb4cbd3+53a1050ad1
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32 import lsst.meas.algorithms as measAlg
33 from lsst.meas.base import SingleFrameMeasurementTask
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 import lsst.utils as utils
37 from .assembleCoadd import (AssembleCoaddTask,
38  CompareWarpAssembleCoaddConfig,
39  CompareWarpAssembleCoaddTask)
40 from .coaddBase import makeSkyInfo
41 from .measurePsf import MeasurePsfTask
42 
43 __all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
44 
45 
46 class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47  dimensions=("tract", "patch", "abstract_filter", "skymap"),
48  defaultTemplates={"inputCoaddName": "deep",
49  "outputCoaddName": "dcr",
50  "warpType": "direct",
51  "warpTypeSuffix": "",
52  "fakesType": ""}):
53  inputWarps = pipeBase.connectionTypes.Input(
54  doc=("Input list of warps to be assembled i.e. stacked."
55  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56  name="{inputCoaddName}Coadd_{warpType}Warp",
57  storageClass="ExposureF",
58  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59  deferLoad=True,
60  multiple=True
61  )
62  skyMap = pipeBase.connectionTypes.Input(
63  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64  name="{inputCoaddName}Coadd_skyMap",
65  storageClass="SkyMap",
66  dimensions=("skymap", ),
67  )
68  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70  " BRIGHT_OBJECT."),
71  name="brightObjectMask",
72  storageClass="ObjectMaskCatalog",
73  dimensions=("tract", "patch", "skymap", "abstract_filter"),
74  )
75  templateExposure = pipeBase.connectionTypes.Input(
76  doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77  name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78  storageClass="ExposureF",
79  dimensions=("tract", "patch", "skymap", "abstract_filter"),
80  )
81  dcrCoadds = pipeBase.connectionTypes.Output(
82  doc="Output coadded exposure, produced by stacking input warps",
83  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84  storageClass="ExposureF",
85  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
86  multiple=True,
87  )
88  dcrNImages = pipeBase.connectionTypes.Output(
89  doc="Output image of number of input images per pixel",
90  name="{outputCoaddName}Coadd_nImage",
91  storageClass="ImageU",
92  dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93  multiple=True,
94  )
95 
96  def __init__(self, *, config=None):
97  super().__init__(config=config)
98  if not config.doWrite:
99  self.outputs.remove("dcrCoadds")
100 
101 
102 class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103  pipelineConnections=DcrAssembleCoaddConnections):
104  dcrNumSubfilters = pexConfig.Field(
105  dtype=int,
106  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107  default=3,
108  )
109  maxNumIter = pexConfig.Field(
110  dtype=int,
111  optional=True,
112  doc="Maximum number of iterations of forward modeling.",
113  default=None,
114  )
115  minNumIter = pexConfig.Field(
116  dtype=int,
117  optional=True,
118  doc="Minimum number of iterations of forward modeling.",
119  default=None,
120  )
121  convergenceThreshold = pexConfig.Field(
122  dtype=float,
123  doc="Target relative change in convergence between iterations of forward modeling.",
124  default=0.001,
125  )
126  useConvergence = pexConfig.Field(
127  dtype=bool,
128  doc="Use convergence test as a forward modeling end condition?"
129  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130  default=True,
131  )
132  baseGain = pexConfig.Field(
133  dtype=float,
134  optional=True,
135  doc="Relative weight to give the new solution vs. the last solution when updating the model."
136  "A value of 1.0 gives equal weight to both solutions."
137  "Small values imply slower convergence of the solution, but can "
138  "help prevent overshooting and failures in the fit."
139  "If ``baseGain`` is None, a conservative gain "
140  "will be calculated from the number of subfilters. ",
141  default=None,
142  )
143  useProgressiveGain = pexConfig.Field(
144  dtype=bool,
145  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146  "When calculating the next gain, we use up to 5 previous gains and convergence values."
147  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148  default=True,
149  )
150  doAirmassWeight = pexConfig.Field(
151  dtype=bool,
152  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153  default=False,
154  )
155  modelWeightsWidth = pexConfig.Field(
156  dtype=float,
157  doc="Width of the region around detected sources to include in the DcrModel.",
158  default=3,
159  )
160  useModelWeights = pexConfig.Field(
161  dtype=bool,
162  doc="Width of the region around detected sources to include in the DcrModel.",
163  default=True,
164  )
165  splitSubfilters = pexConfig.Field(
166  dtype=bool,
167  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168  "Instead of at the midpoint",
169  default=True,
170  )
171  splitThreshold = pexConfig.Field(
172  dtype=float,
173  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174  "Set to 0 to always split the subfilters.",
175  default=0.1,
176  )
177  regularizeModelIterations = pexConfig.Field(
178  dtype=float,
179  doc="Maximum relative change of the model allowed between iterations."
180  "Set to zero to disable.",
181  default=2.,
182  )
183  regularizeModelFrequency = pexConfig.Field(
184  dtype=float,
185  doc="Maximum relative change of the model allowed between subfilters."
186  "Set to zero to disable.",
187  default=4.,
188  )
189  convergenceMaskPlanes = pexConfig.ListField(
190  dtype=str,
191  default=["DETECTED"],
192  doc="Mask planes to use to calculate convergence."
193  )
194  regularizationWidth = pexConfig.Field(
195  dtype=int,
196  default=2,
197  doc="Minimum radius of a region to include in regularization, in pixels."
198  )
199  imageInterpOrder = pexConfig.Field(
200  dtype=int,
201  doc="The order of the spline interpolation used to shift the image plane.",
202  default=3,
203  )
204  accelerateModel = pexConfig.Field(
205  dtype=float,
206  doc="Factor to amplify the differences between model planes by to speed convergence.",
207  default=3,
208  )
209  doCalculatePsf = pexConfig.Field(
210  dtype=bool,
211  doc="Set to detect stars and recalculate the PSF from the final coadd."
212  "Otherwise the PSF is estimated from a selection of the best input exposures",
213  default=False,
214  )
215  detectPsfSources = pexConfig.ConfigurableField(
216  target=measAlg.SourceDetectionTask,
217  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218  )
219  measurePsfSources = pexConfig.ConfigurableField(
220  target=SingleFrameMeasurementTask,
221  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222  )
223  measurePsf = pexConfig.ConfigurableField(
224  target=MeasurePsfTask,
225  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226  )
227 
228  def setDefaults(self):
229  CompareWarpAssembleCoaddConfig.setDefaults(self)
230  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
231  self.doNImage = True
232  self.assembleStaticSkyModel.warpType = self.warpType
233  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
234  self.assembleStaticSkyModel.doNImage = False
235  self.assembleStaticSkyModel.doWrite = False
236  self.detectPsfSources.returnOriginalFootprints = False
237  self.detectPsfSources.thresholdPolarity = "positive"
238  # Only use bright sources for PSF measurement
239  self.detectPsfSources.thresholdValue = 50
240  self.detectPsfSources.nSigmaToGrow = 2
241  # A valid star for PSF measurement should at least fill 5x5 pixels
242  self.detectPsfSources.minPixels = 25
243  # Use the variance plane to calculate signal to noise
244  self.detectPsfSources.thresholdType = "pixel_stdev"
245  # The signal to noise limit is good enough, while the flux limit is set
246  # in dimensionless units and may not be appropriate for all data sets.
247  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
248 
249 
250 class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
251  """Assemble DCR coadded images from a set of warps.
252 
253  Attributes
254  ----------
255  bufferSize : `int`
256  The number of pixels to grow each subregion by to allow for DCR.
257 
258  Notes
259  -----
260  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
261  Warps (also called coadded temporary exposures), including the effects of
262  Differential Chromatic Refraction (DCR).
263  For full details of the mathematics and algorithm, please see
264  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
265 
266  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
267  each subfilter used in the iterative calculation.
268  It begins by dividing the bandpass-defining filter into N equal bandwidth
269  "subfilters", and divides the flux in each pixel from an initial coadd
270  equally into each as a "dcrModel". Because the airmass and parallactic
271  angle of each individual exposure is known, we can calculate the shift
272  relative to the center of the band in each subfilter due to DCR. For each
273  exposure we apply this shift as a linear transformation to the dcrModels
274  and stack the results to produce a DCR-matched exposure. The matched
275  exposures are subtracted from the input exposures to produce a set of
276  residual images, and these residuals are reverse shifted for each
277  exposures' subfilters and stacked. The shifted and stacked residuals are
278  added to the dcrModels to produce a new estimate of the flux in each pixel
279  within each subfilter. The dcrModels are solved for iteratively, which
280  continues until the solution from a new iteration improves by less than
281  a set percentage, or a maximum number of iterations is reached.
282  Two forms of regularization are employed to reduce unphysical results.
283  First, the new solution is averaged with the solution from the previous
284  iteration, which mitigates oscillating solutions where the model
285  overshoots with alternating very high and low values.
286  Second, a common degeneracy when the data have a limited range of airmass or
287  parallactic angle values is for one subfilter to be fit with very low or
288  negative values, while another subfilter is fit with very high values. This
289  typically appears in the form of holes next to sources in one subfilter,
290  and corresponding extended wings in another. Because each subfilter has
291  a narrow bandwidth we assume that physical sources that are above the noise
292  level will not vary in flux by more than a factor of `frequencyClampFactor`
293  between subfilters, and pixels that have flux deviations larger than that
294  factor will have the excess flux distributed evenly among all subfilters.
295  If `splitSubfilters` is set, then each subfilter will be further sub-
296  divided during the forward modeling step (only). This approximates using
297  a higher number of subfilters that may be necessary for high airmass
298  observations, but does not increase the number of free parameters in the
299  fit. This is needed when there are high airmass observations which would
300  otherwise have significant DCR even within a subfilter. Because calculating
301  the shifted images takes most of the time, splitting the subfilters is
302  turned off by way of the `splitThreshold` option for low-airmass
303  observations that do not suffer from DCR within a subfilter.
304  """
305 
306  ConfigClass = DcrAssembleCoaddConfig
307  _DefaultName = "dcrAssembleCoadd"
308 
309  def __init__(self, *args, **kwargs):
310  super().__init__(*args, **kwargs)
311  if self.config.doCalculatePsf:
312  self.schema = afwTable.SourceTable.makeMinimalSchema()
313  self.makeSubtask("detectPsfSources", schema=self.schema)
314  self.makeSubtask("measurePsfSources", schema=self.schema)
315  self.makeSubtask("measurePsf", schema=self.schema)
316 
317  @utils.inheritDoc(pipeBase.PipelineTask)
318  def runQuantum(self, butlerQC, inputRefs, outputRefs):
319  # Docstring to be formatted with info from PipelineTask.runQuantum
320  """
321  Notes
322  -----
323  Assemble a coadd from a set of Warps.
324 
325  PipelineTask (Gen3) entry point to Coadd a set of Warps.
326  Analogous to `runDataRef`, it prepares all the data products to be
327  passed to `run`, and processes the results before returning a struct
328  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
329  Therefore, its inputs are accessed subregion by subregion
330  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
331  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
332  correspond to an update in `runDataRef` while both entry points
333  are used.
334  """
335  inputData = butlerQC.get(inputRefs)
336 
337  # Construct skyInfo expected by run
338  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
339  skyMap = inputData["skyMap"]
340  outputDataId = butlerQC.quantum.dataId
341 
342  inputData['skyInfo'] = makeSkyInfo(skyMap,
343  tractId=outputDataId['tract'],
344  patchId=outputDataId['patch'])
345 
346  # Construct list of input Deferred Datasets
347  # These quack a bit like like Gen2 DataRefs
348  warpRefList = inputData['inputWarps']
349  # Perform same middle steps as `runDataRef` does
350  inputs = self.prepareInputs(warpRefList)
351  self.log.info("Found %d %s", len(inputs.tempExpRefList),
352  self.getTempExpDatasetName(self.warpType))
353  if len(inputs.tempExpRefList) == 0:
354  self.log.warn("No coadd temporary exposures found")
355  return
356 
357  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
358  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
359  inputs.weightList, supplementaryData=supplementaryData)
360 
361  inputData.setdefault('brightObjectMask', None)
362  for subfilter in range(self.config.dcrNumSubfilters):
363  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
364  retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
365  self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
366 
367  if self.config.doWrite:
368  butlerQC.put(retStruct, outputRefs)
369  return retStruct
370 
371  @pipeBase.timeMethod
372  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
373  """Assemble a coadd from a set of warps.
374 
375  Coadd a set of Warps. Compute weights to be applied to each Warp and
376  find scalings to match the photometric zeropoint to a reference Warp.
377  Assemble the Warps using run method.
378  Forward model chromatic effects across multiple subfilters,
379  and subtract from the input Warps to build sets of residuals.
380  Use the residuals to construct a new ``DcrModel`` for each subfilter,
381  and iterate until the model converges.
382  Interpolate over NaNs and optionally write the coadd to disk.
383  Return the coadded exposure.
384 
385  Parameters
386  ----------
387  dataRef : `lsst.daf.persistence.ButlerDataRef`
388  Data reference defining the patch for coaddition and the
389  reference Warp
390  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
391  List of data references to warps. Data to be coadded will be
392  selected from this list based on overlap with the patch defined by
393  the data reference.
394 
395  Returns
396  -------
397  results : `lsst.pipe.base.Struct`
398  The Struct contains the following fields:
399 
400  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
401  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
402  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
403  - ``dcrNImages``: `list` of exposure count images for each subfilter
404  """
405  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
406  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
407 
408  skyInfo = self.getSkyInfo(dataRef)
409  if warpRefList is None:
410  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
411  if len(calExpRefList) == 0:
412  self.log.warn("No exposures to coadd")
413  return
414  self.log.info("Coadding %d exposures", len(calExpRefList))
415 
416  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
417 
418  inputData = self.prepareInputs(warpRefList)
419  self.log.info("Found %d %s", len(inputData.tempExpRefList),
420  self.getTempExpDatasetName(self.warpType))
421  if len(inputData.tempExpRefList) == 0:
422  self.log.warn("No coadd temporary exposures found")
423  return
424 
425  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
426 
427  results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
428  inputData.weightList, supplementaryData=supplementaryData)
429  if results is None:
430  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
431  skyInfo.patchInfo.getIndex())
432  return
433 
434  if self.config.doCalculatePsf:
435  self.measureCoaddPsf(results.coaddExposure)
436  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
437  for subfilter in range(self.config.dcrNumSubfilters):
438  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
439  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
440  self.processResults(results.dcrCoadds[subfilter],
441  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
442  if self.config.doWrite:
443  self.log.info("Persisting dcrCoadd")
444  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
445  numSubfilters=self.config.dcrNumSubfilters)
446  if self.config.doNImage and results.dcrNImages is not None:
447  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
448  numSubfilters=self.config.dcrNumSubfilters)
449 
450  return results
451 
452  @utils.inheritDoc(AssembleCoaddTask)
453  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
454  """Load the previously-generated template coadd.
455 
456  This can be removed entirely once we no longer support the Gen 2 butler.
457 
458  Returns
459  -------
460  templateCoadd : `lsst.pipe.base.Struct`
461  Result struct with components:
462 
463  - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
464  """
465  templateCoadd = butlerQC.get(inputRefs.templateExposure)
466 
467  return pipeBase.Struct(templateCoadd=templateCoadd)
468 
469  def measureCoaddPsf(self, coaddExposure):
470  """Detect sources on the coadd exposure and measure the final PSF.
471 
472  Parameters
473  ----------
474  coaddExposure : `lsst.afw.image.Exposure`
475  The final coadded exposure.
476  """
477  table = afwTable.SourceTable.make(self.schema)
478  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
479  coaddSources = detResults.sources
480  self.measurePsfSources.run(
481  measCat=coaddSources,
482  exposure=coaddExposure
483  )
484  # Measure the PSF on the stacked subfilter coadds if possible.
485  # We should already have a decent estimate of the coadd PSF, however,
486  # so in case of any errors simply log them as a warning and use the
487  # default PSF.
488  try:
489  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
490  except Exception as e:
491  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
492  else:
493  coaddExposure.setPsf(psfResults.psf)
494 
495  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
496  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
497 
498  Sets the property ``bufferSize``.
499 
500  Parameters
501  ----------
502  templateCoadd : `lsst.afw.image.ExposureF`
503  The initial coadd exposure before accounting for DCR.
504  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
505  `lsst.daf.persistence.ButlerDataRef`
506  The data references to the input warped exposures.
507  weightList : `list` of `float`
508  The weight to give each input exposure in the coadd
509  Will be modified in place if ``doAirmassWeight`` is set.
510 
511  Returns
512  -------
513  dcrModels : `lsst.pipe.tasks.DcrModel`
514  Best fit model of the true sky after correcting chromatic effects.
515 
516  Raises
517  ------
518  NotImplementedError
519  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
520  """
521  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
522  filterInfo = templateCoadd.getFilter()
523  if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
524  raise NotImplementedError("No minimum/maximum wavelength information found"
525  " in the filter definition! Please add lambdaMin and lambdaMax"
526  " to the Mapper class in your obs package.")
527  tempExpName = self.getTempExpDatasetName(self.warpType)
528  dcrShifts = []
529  airmassDict = {}
530  angleDict = {}
531  psfSizeDict = {}
532  for visitNum, warpExpRef in enumerate(warpRefList):
533  if isinstance(warpExpRef, DeferredDatasetHandle):
534  # Gen 3 API
535  visitInfo = warpExpRef.get(component="visitInfo")
536  psf = warpExpRef.get(component="psf")
537  else:
538  # Gen 2 API. Delete this when Gen 2 retired
539  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
540  psf = warpExpRef.get(tempExpName).getPsf()
541  visit = warpExpRef.dataId["visit"]
542  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
543  airmass = visitInfo.getBoresightAirmass()
544  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
545  airmassDict[visit] = airmass
546  angleDict[visit] = parallacticAngle
547  psfSizeDict[visit] = psfSize
548  if self.config.doAirmassWeight:
549  weightList[visitNum] *= airmass
550  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
551  filterInfo, self.config.dcrNumSubfilters))))
552  self.log.info("Selected airmasses:\n%s", airmassDict)
553  self.log.info("Selected parallactic angles:\n%s", angleDict)
554  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
555  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
556  try:
557  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
558  except Exception as e:
559  self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e)
560  else:
561  psf = templateCoadd.getPsf()
562  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
563  self.config.dcrNumSubfilters,
564  filterInfo=filterInfo,
565  psf=psf)
566  return dcrModels
567 
568  @pipeBase.timeMethod
569  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
570  supplementaryData=None):
571  """Assemble the coadd.
572 
573  Requires additional inputs Struct ``supplementaryData`` to contain a
574  ``templateCoadd`` that serves as the model of the static sky.
575 
576  Find artifacts and apply them to the warps' masks creating a list of
577  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
578  Then pass these alternative masks to the base class's assemble method.
579 
580  Divide the ``templateCoadd`` evenly between each subfilter of a
581  ``DcrModel`` as the starting best estimate of the true wavelength-
582  dependent sky. Forward model the ``DcrModel`` using the known
583  chromatic effects in each subfilter and calculate a convergence metric
584  based on how well the modeled template matches the input warps. If
585  the convergence has not yet reached the desired threshold, then shift
586  and stack the residual images to build a new ``DcrModel``. Apply
587  conditioning to prevent oscillating solutions between iterations or
588  between subfilters.
589 
590  Once the ``DcrModel`` reaches convergence or the maximum number of
591  iterations has been reached, fill the metadata for each subfilter
592  image and make them proper ``coaddExposure``s.
593 
594  Parameters
595  ----------
596  skyInfo : `lsst.pipe.base.Struct`
597  Patch geometry information, from getSkyInfo
598  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
599  `lsst.daf.persistence.ButlerDataRef`
600  The data references to the input warped exposures.
601  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
602  The image scalars correct for the zero point of the exposures.
603  weightList : `list` of `float`
604  The weight to give each input exposure in the coadd
605  supplementaryData : `lsst.pipe.base.Struct`
606  Result struct returned by ``makeSupplementaryData`` with components:
607 
608  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
609 
610  Returns
611  -------
612  result : `lsst.pipe.base.Struct`
613  Result struct with components:
614 
615  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
616  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
617  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
618  - ``dcrNImages``: `list` of exposure count images for each subfilter
619  """
620  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
621  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
622  templateCoadd = supplementaryData.templateCoadd
623  baseMask = templateCoadd.mask.clone()
624  # The variance plane is for each subfilter
625  # and should be proportionately lower than the full-band image
626  baseVariance = templateCoadd.variance.clone()
627  baseVariance /= self.config.dcrNumSubfilters
628  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
629  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
630  templateCoadd.setMask(baseMask)
631  badMaskPlanes = self.config.badMaskPlanes[:]
632  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
633  # This is because pixels in observations that are significantly affect by DCR
634  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
635  # but those are necessary to constrain the DCR model.
636  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
637 
638  stats = self.prepareStats(mask=badPixelMask)
639  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
640  if self.config.doNImage:
641  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
642  spanSetMaskList, stats.ctrl)
643  nImage = afwImage.ImageU(skyInfo.bbox)
644  # Note that this nImage will be a factor of dcrNumSubfilters higher than
645  # the nImage returned by assembleCoadd for most pixels. This is because each
646  # subfilter may have a different nImage, and fractional values are not allowed.
647  for dcrNImage in dcrNImages:
648  nImage += dcrNImage
649  else:
650  dcrNImages = None
651 
652  subregionSize = geom.Extent2I(*self.config.subregionSize)
653  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
654  * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
655  subIter = 0
656  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
657  modelIter = 0
658  subIter += 1
659  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
660  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
661  dcrBBox = geom.Box2I(subBBox)
662  dcrBBox.grow(self.bufferSize)
663  dcrBBox.clip(dcrModels.bbox)
664  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
665  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
666  imageScalerList, spanSetMaskList)
667  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
668  warpRefList, weightList, stats.ctrl)
669  self.log.info("Initial convergence : %s", convergenceMetric)
670  convergenceList = [convergenceMetric]
671  gainList = []
672  convergenceCheck = 1.
673  refImage = templateCoadd.image
674  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
675  gain = self.calculateGain(convergenceList, gainList)
676  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
677  stats.ctrl, convergenceMetric, gain,
678  modelWeights, refImage, dcrWeights)
679  if self.config.useConvergence:
680  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
681  warpRefList, weightList, stats.ctrl)
682  if convergenceMetric == 0:
683  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
684  "most likely due to there being no valid data in the region.",
685  skyInfo.patchInfo.getIndex(), subIter)
686  break
687  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
688  if (convergenceCheck < 0) & (modelIter > minNumIter):
689  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
690  "iterations or desired convergence improvement of %s."
691  " Divergence: %s",
692  skyInfo.patchInfo.getIndex(), subIter,
693  self.config.convergenceThreshold, convergenceCheck)
694  break
695  convergenceList.append(convergenceMetric)
696  if modelIter > maxNumIter:
697  if self.config.useConvergence:
698  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
699  "before reaching desired convergence improvement of %s."
700  " Final convergence improvement: %s",
701  skyInfo.patchInfo.getIndex(), subIter,
702  self.config.convergenceThreshold, convergenceCheck)
703  break
704 
705  if self.config.useConvergence:
706  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
707  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
708  modelIter += 1
709  else:
710  if self.config.useConvergence:
711  self.log.info("Coadd patch %s subregion %s finished with "
712  "convergence metric %s after %s iterations",
713  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
714  else:
715  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
716  skyInfo.patchInfo.getIndex(), subIter, modelIter)
717  if self.config.useConvergence and convergenceMetric > 0:
718  self.log.info("Final convergence improvement was %.4f%% overall",
719  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
720 
721  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
722  calibration=self.scaleZeroPoint.getPhotoCalib(),
723  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
724  mask=baseMask,
725  variance=baseVariance)
726  coaddExposure = self.stackCoadd(dcrCoadds)
727  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
728  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
729 
730  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
731  """Calculate the number of exposures contributing to each subfilter.
732 
733  Parameters
734  ----------
735  dcrModels : `lsst.pipe.tasks.DcrModel`
736  Best fit model of the true sky after correcting chromatic effects.
737  bbox : `lsst.geom.box.Box2I`
738  Bounding box of the patch to coadd.
739  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
740  `lsst.daf.persistence.ButlerDataRef`
741  The data references to the input warped exposures.
742  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
743  Each element of the `dict` contains the new mask plane name
744  (e.g. "CLIPPED and/or "NO_DATA") as the key,
745  and the list of SpanSets to apply to the mask.
746  statsCtrl : `lsst.afw.math.StatisticsControl`
747  Statistics control object for coadd
748 
749  Returns
750  -------
751  dcrNImages : `list` of `lsst.afw.image.ImageU`
752  List of exposure count images for each subfilter
753  dcrWeights : `list` of `lsst.afw.image.ImageF`
754  Per-pixel weights for each subfilter.
755  Equal to 1/(number of unmasked images contributing to each pixel).
756  """
757  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
758  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
759  tempExpName = self.getTempExpDatasetName(self.warpType)
760  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
761  if isinstance(warpExpRef, DeferredDatasetHandle):
762  # Gen 3 API
763  exposure = warpExpRef.get(parameters={'bbox': bbox})
764  else:
765  # Gen 2 API. Delete this when Gen 2 retired
766  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
767  visitInfo = exposure.getInfo().getVisitInfo()
768  wcs = exposure.getInfo().getWcs()
769  mask = exposure.mask
770  if altMaskSpans is not None:
771  self.applyAltMaskPlanes(mask, altMaskSpans)
772  weightImage = np.zeros_like(exposure.image.array)
773  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
774  # The weights must be shifted in exactly the same way as the residuals,
775  # because they will be used as the denominator in the weighted average of residuals.
776  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
777  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
778  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
779  dcrWeight.array += shiftedWeights
780  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
781  weightsThreshold = 1.
782  goodPix = dcrWeights[0].array > weightsThreshold
783  for weights in dcrWeights[1:]:
784  goodPix = (weights.array > weightsThreshold) & goodPix
785  for subfilter in range(self.config.dcrNumSubfilters):
786  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
787  dcrWeights[subfilter].array[~goodPix] = 0.
788  dcrNImages[subfilter].array[~goodPix] = 0
789  return (dcrNImages, dcrWeights)
790 
791  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
792  statsCtrl, convergenceMetric,
793  gain, modelWeights, refImage, dcrWeights):
794  """Assemble the DCR coadd for a sub-region.
795 
796  Build a DCR-matched template for each input exposure, then shift the
797  residuals according to the DCR in each subfilter.
798  Stack the shifted residuals and apply them as a correction to the
799  solution from the previous iteration.
800  Restrict the new model solutions from varying by more than a factor of
801  `modelClampFactor` from the last solution, and additionally restrict the
802  individual subfilter models from varying by more than a factor of
803  `frequencyClampFactor` from their average.
804  Finally, mitigate potentially oscillating solutions by averaging the new
805  solution with the solution from the previous iteration, weighted by
806  their convergence metric.
807 
808  Parameters
809  ----------
810  dcrModels : `lsst.pipe.tasks.DcrModel`
811  Best fit model of the true sky after correcting chromatic effects.
812  subExposures : `dict` of `lsst.afw.image.ExposureF`
813  The pre-loaded exposures for the current subregion.
814  bbox : `lsst.geom.box.Box2I`
815  Bounding box of the subregion to coadd.
816  dcrBBox : `lsst.geom.box.Box2I`
817  Sub-region of the coadd which includes a buffer to allow for DCR.
818  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
819  `lsst.daf.persistence.ButlerDataRef`
820  The data references to the input warped exposures.
821  statsCtrl : `lsst.afw.math.StatisticsControl`
822  Statistics control object for coadd
823  convergenceMetric : `float`
824  Quality of fit metric for the matched templates of the input images.
825  gain : `float`, optional
826  Relative weight to give the new solution when updating the model.
827  modelWeights : `numpy.ndarray` or `float`
828  A 2D array of weight values that tapers smoothly to zero away from detected sources.
829  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
830  refImage : `lsst.afw.image.Image`
831  A reference image used to supply the default pixel values.
832  dcrWeights : `list` of `lsst.afw.image.Image`
833  Per-pixel weights for each subfilter.
834  Equal to 1/(number of unmasked images contributing to each pixel).
835  """
836  residualGeneratorList = []
837 
838  for warpExpRef in warpRefList:
839  visit = warpExpRef.dataId["visit"]
840  exposure = subExposures[visit]
841  visitInfo = exposure.getInfo().getVisitInfo()
842  wcs = exposure.getInfo().getWcs()
843  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
844  order=self.config.imageInterpOrder,
845  splitSubfilters=self.config.splitSubfilters,
846  splitThreshold=self.config.splitThreshold,
847  amplifyModel=self.config.accelerateModel)
848  residual = exposure.image.array - templateImage.array
849  # Note that the variance plane here is used to store weights, not the actual variance
850  residual *= exposure.variance.array
851  # The residuals are stored as a list of generators.
852  # This allows the residual for a given subfilter and exposure to be created
853  # on the fly, instead of needing to store them all in memory.
854  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
855 
856  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
857  gain=gain,
858  modelWeights=modelWeights,
859  refImage=refImage,
860  dcrWeights=dcrWeights)
861  dcrModels.assign(dcrSubModelOut, bbox)
862 
863  def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
864  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
865 
866  Parameters
867  ----------
868  residual : `numpy.ndarray`
869  The residual masked image for one exposure,
870  after subtracting the matched template
871  visitInfo : `lsst.afw.image.VisitInfo`
872  Metadata for the exposure.
873  wcs : `lsst.afw.geom.SkyWcs`
874  Coordinate system definition (wcs) for the exposure.
875  filterInfo : `lsst.afw.image.Filter`
876  The filter definition, set in the current instruments' obs package.
877  Required for any calculation of DCR, including making matched templates.
878 
879  Yields
880  ------
881  residualImage : `numpy.ndarray`
882  The residual image for the next subfilter, shifted for DCR.
883  """
884  # Pre-calculate the spline-filtered residual image, so that step can be
885  # skipped in the shift calculation in `applyDcr`.
886  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
887  # Note that `splitSubfilters` is always turned off in the reverse direction.
888  # This option introduces additional blurring if applied to the residuals.
889  dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
890  splitSubfilters=False)
891  for dcr in dcrShift:
892  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
893  doPrefilter=False, order=self.config.imageInterpOrder)
894 
895  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
896  gain, modelWeights, refImage, dcrWeights):
897  """Calculate a new DcrModel from a set of image residuals.
898 
899  Parameters
900  ----------
901  dcrModels : `lsst.pipe.tasks.DcrModel`
902  Current model of the true sky after correcting chromatic effects.
903  residualGeneratorList : `generator` of `numpy.ndarray`
904  The residual image for the next subfilter, shifted for DCR.
905  dcrBBox : `lsst.geom.box.Box2I`
906  Sub-region of the coadd which includes a buffer to allow for DCR.
907  statsCtrl : `lsst.afw.math.StatisticsControl`
908  Statistics control object for coadd
909  gain : `float`
910  Relative weight to give the new solution when updating the model.
911  modelWeights : `numpy.ndarray` or `float`
912  A 2D array of weight values that tapers smoothly to zero away from detected sources.
913  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
914  refImage : `lsst.afw.image.Image`
915  A reference image used to supply the default pixel values.
916  dcrWeights : `list` of `lsst.afw.image.Image`
917  Per-pixel weights for each subfilter.
918  Equal to 1/(number of unmasked images contributing to each pixel).
919 
920  Returns
921  -------
922  dcrModel : `lsst.pipe.tasks.DcrModel`
923  New model of the true sky after correcting chromatic effects.
924  """
925  newModelImages = []
926  for subfilter, model in enumerate(dcrModels):
927  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
928  residual = np.sum(residualsList, axis=0)
929  residual *= dcrWeights[subfilter][dcrBBox].array
930  # `MaskedImage`s only support in-place addition, so rename for readability
931  newModel = model[dcrBBox].clone()
932  newModel.array += residual
933  # Catch any invalid values
934  badPixels = ~np.isfinite(newModel.array)
935  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
936  if self.config.regularizeModelIterations > 0:
937  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
938  self.config.regularizeModelIterations,
939  self.config.regularizationWidth)
940  newModelImages.append(newModel)
941  if self.config.regularizeModelFrequency > 0:
942  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
943  self.config.regularizeModelFrequency,
944  self.config.regularizationWidth)
945  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
946  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
947  return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
948  dcrModels.mask, dcrModels.variance)
949 
950  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
951  """Calculate a quality of fit metric for the matched templates.
952 
953  Parameters
954  ----------
955  dcrModels : `lsst.pipe.tasks.DcrModel`
956  Best fit model of the true sky after correcting chromatic effects.
957  subExposures : `dict` of `lsst.afw.image.ExposureF`
958  The pre-loaded exposures for the current subregion.
959  bbox : `lsst.geom.box.Box2I`
960  Sub-region to coadd
961  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
962  `lsst.daf.persistence.ButlerDataRef`
963  The data references to the input warped exposures.
964  weightList : `list` of `float`
965  The weight to give each input exposure in the coadd
966  statsCtrl : `lsst.afw.math.StatisticsControl`
967  Statistics control object for coadd
968 
969  Returns
970  -------
971  convergenceMetric : `float`
972  Quality of fit metric for all input exposures, within the sub-region
973  """
974  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
975  nSigma = 3.
976  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
977  bufferSize=self.bufferSize)
978  if np.max(significanceImage) == 0:
979  significanceImage += 1.
980  weight = 0
981  metric = 0.
982  metricList = {}
983  for warpExpRef, expWeight in zip(warpRefList, weightList):
984  visit = warpExpRef.dataId["visit"]
985  exposure = subExposures[visit][bbox]
986  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
987  metric += singleMetric
988  metricList[visit] = singleMetric
989  weight += 1.
990  self.log.info("Individual metrics:\n%s", metricList)
991  return 1.0 if weight == 0.0 else metric/weight
992 
993  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
994  """Calculate a quality of fit metric for a single matched template.
995 
996  Parameters
997  ----------
998  dcrModels : `lsst.pipe.tasks.DcrModel`
999  Best fit model of the true sky after correcting chromatic effects.
1000  exposure : `lsst.afw.image.ExposureF`
1001  The input warped exposure to evaluate.
1002  significanceImage : `numpy.ndarray`
1003  Array of weights for each pixel corresponding to its significance
1004  for the convergence calculation.
1005  statsCtrl : `lsst.afw.math.StatisticsControl`
1006  Statistics control object for coadd
1007 
1008  Returns
1009  -------
1010  convergenceMetric : `float`
1011  Quality of fit metric for one exposure, within the sub-region.
1012  """
1013  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1014  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1015  order=self.config.imageInterpOrder,
1016  splitSubfilters=self.config.splitSubfilters,
1017  splitThreshold=self.config.splitThreshold,
1018  amplifyModel=self.config.accelerateModel)
1019  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1020  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1021 
1022  finitePixels = np.isfinite(diffVals)
1023  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1024  convergeMaskPixels = exposure.mask.array & convergeMask > 0
1025  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1026  if np.sum(usePixels) == 0:
1027  metric = 0.
1028  else:
1029  diffUse = diffVals[usePixels]
1030  refUse = refVals[usePixels]
1031  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1032  return metric
1033 
1034  def stackCoadd(self, dcrCoadds):
1035  """Add a list of sub-band coadds together.
1036 
1037  Parameters
1038  ----------
1039  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1040  A list of coadd exposures, each exposure containing
1041  the model for one subfilter.
1042 
1043  Returns
1044  -------
1045  coaddExposure : `lsst.afw.image.ExposureF`
1046  A single coadd exposure that is the sum of the sub-bands.
1047  """
1048  coaddExposure = dcrCoadds[0].clone()
1049  for coadd in dcrCoadds[1:]:
1050  coaddExposure.maskedImage += coadd.maskedImage
1051  return coaddExposure
1052 
1053  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1054  mask=None, variance=None):
1055  """Create a list of coadd exposures from a list of masked images.
1056 
1057  Parameters
1058  ----------
1059  dcrModels : `lsst.pipe.tasks.DcrModel`
1060  Best fit model of the true sky after correcting chromatic effects.
1061  skyInfo : `lsst.pipe.base.Struct`
1062  Patch geometry information, from getSkyInfo
1063  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1064  `lsst.daf.persistence.ButlerDataRef`
1065  The data references to the input warped exposures.
1066  weightList : `list` of `float`
1067  The weight to give each input exposure in the coadd
1068  calibration : `lsst.afw.Image.PhotoCalib`, optional
1069  Scale factor to set the photometric calibration of an exposure.
1070  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1071  A record of the observations that are included in the coadd.
1072  mask : `lsst.afw.image.Mask`, optional
1073  Optional mask to override the values in the final coadd.
1074  variance : `lsst.afw.image.Image`, optional
1075  Optional variance plane to override the values in the final coadd.
1076 
1077  Returns
1078  -------
1079  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1080  A list of coadd exposures, each exposure containing
1081  the model for one subfilter.
1082  """
1083  dcrCoadds = []
1084  refModel = dcrModels.getReferenceImage()
1085  for model in dcrModels:
1086  if self.config.accelerateModel > 1:
1087  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1088  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1089  if calibration is not None:
1090  coaddExposure.setPhotoCalib(calibration)
1091  if coaddInputs is not None:
1092  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1093  # Set the metadata for the coadd, including PSF and aperture corrections.
1094  self.assembleMetadata(coaddExposure, warpRefList, weightList)
1095  # Overwrite the PSF
1096  coaddExposure.setPsf(dcrModels.psf)
1097  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1098  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1099  maskedImage.image = model
1100  maskedImage.mask = dcrModels.mask
1101  maskedImage.variance = dcrModels.variance
1102  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1103  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1104  if mask is not None:
1105  coaddExposure.setMask(mask)
1106  if variance is not None:
1107  coaddExposure.setVariance(variance)
1108  dcrCoadds.append(coaddExposure)
1109  return dcrCoadds
1110 
1111  def calculateGain(self, convergenceList, gainList):
1112  """Calculate the gain to use for the current iteration.
1113 
1114  After calculating a new DcrModel, each value is averaged with the
1115  value in the corresponding pixel from the previous iteration. This
1116  reduces oscillating solutions that iterative techniques are plagued by,
1117  and speeds convergence. By far the biggest changes to the model
1118  happen in the first couple iterations, so we can also use a more
1119  aggressive gain later when the model is changing slowly.
1120 
1121  Parameters
1122  ----------
1123  convergenceList : `list` of `float`
1124  The quality of fit metric from each previous iteration.
1125  gainList : `list` of `float`
1126  The gains used in each previous iteration: appended with the new
1127  gain value.
1128  Gains are numbers between ``self.config.baseGain`` and 1.
1129 
1130  Returns
1131  -------
1132  gain : `float`
1133  Relative weight to give the new solution when updating the model.
1134  A value of 1.0 gives equal weight to both solutions.
1135 
1136  Raises
1137  ------
1138  ValueError
1139  If ``len(convergenceList) != len(gainList)+1``.
1140  """
1141  nIter = len(convergenceList)
1142  if nIter != len(gainList) + 1:
1143  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1144  % (len(convergenceList), len(gainList)))
1145 
1146  if self.config.baseGain is None:
1147  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1148  # The more subfilters being modeled, the lower the gain should be.
1149  baseGain = 1./(self.config.dcrNumSubfilters - 1)
1150  else:
1151  baseGain = self.config.baseGain
1152 
1153  if self.config.useProgressiveGain and nIter > 2:
1154  # To calculate the best gain to use, compare the past gains that have been used
1155  # with the resulting convergences to estimate the best gain to use.
1156  # Algorithmically, this is a Kalman filter.
1157  # If forward modeling proceeds perfectly, the convergence metric should
1158  # asymptotically approach a final value.
1159  # We can estimate that value from the measured changes in convergence
1160  # weighted by the gains used in each previous iteration.
1161  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1162  for i in range(nIter - 1)]
1163  # The convergence metric is strictly positive, so if the estimated final convergence is
1164  # less than zero, force it to zero.
1165  estFinalConv = np.array(estFinalConv)
1166  estFinalConv[estFinalConv < 0] = 0
1167  # Because the estimate may slowly change over time, only use the most recent measurements.
1168  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1169  lastGain = gainList[-1]
1170  lastConv = convergenceList[-2]
1171  newConv = convergenceList[-1]
1172  # The predicted convergence is the value we would get if the new model calculated
1173  # in the previous iteration was perfect. Recall that the updated model that is
1174  # actually used is the gain-weighted average of the new and old model,
1175  # so the convergence would be similarly weighted.
1176  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1177  # If the measured and predicted convergence are very close, that indicates
1178  # that our forward model is accurate and we can use a more aggressive gain
1179  # If the measured convergence is significantly worse (or better!) than predicted,
1180  # that indicates that the model is not converging as expected and
1181  # we should use a more conservative gain.
1182  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1183  newGain = 1 - abs(delta)
1184  # Average the gains to prevent oscillating solutions.
1185  newGain = (newGain + lastGain)/2.
1186  gain = max(baseGain, newGain)
1187  else:
1188  gain = baseGain
1189  gainList.append(gain)
1190  return gain
1191 
1192  def calculateModelWeights(self, dcrModels, dcrBBox):
1193  """Build an array that smoothly tapers to 0 away from detected sources.
1194 
1195  Parameters
1196  ----------
1197  dcrModels : `lsst.pipe.tasks.DcrModel`
1198  Best fit model of the true sky after correcting chromatic effects.
1199  dcrBBox : `lsst.geom.box.Box2I`
1200  Sub-region of the coadd which includes a buffer to allow for DCR.
1201 
1202  Returns
1203  -------
1204  weights : `numpy.ndarray` or `float`
1205  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1206  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1207 
1208  Raises
1209  ------
1210  ValueError
1211  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1212  """
1213  if not self.config.useModelWeights:
1214  return 1.0
1215  if self.config.modelWeightsWidth < 0:
1216  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1217  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1218  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1219  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1220  weights[convergeMaskPixels] = 1.
1221  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1222  weights /= np.max(weights)
1223  return weights
1224 
1225  def applyModelWeights(self, modelImages, refImage, modelWeights):
1226  """Smoothly replace model pixel values with those from a
1227  reference at locations away from detected sources.
1228 
1229  Parameters
1230  ----------
1231  modelImages : `list` of `lsst.afw.image.Image`
1232  The new DCR model images from the current iteration.
1233  The values will be modified in place.
1234  refImage : `lsst.afw.image.MaskedImage`
1235  A reference image used to supply the default pixel values.
1236  modelWeights : `numpy.ndarray` or `float`
1237  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1238  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1239  """
1240  if self.config.useModelWeights:
1241  for model in modelImages:
1242  model.array *= modelWeights
1243  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1244 
1245  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1246  """Pre-load sub-regions of a list of exposures.
1247 
1248  Parameters
1249  ----------
1250  bbox : `lsst.geom.box.Box2I`
1251  Sub-region to coadd
1252  statsCtrl : `lsst.afw.math.StatisticsControl`
1253  Statistics control object for coadd
1254  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1255  `lsst.daf.persistence.ButlerDataRef`
1256  The data references to the input warped exposures.
1257  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1258  The image scalars correct for the zero point of the exposures.
1259  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1260  Each element is dict with keys = mask plane name to add the spans to
1261 
1262  Returns
1263  -------
1264  subExposures : `dict`
1265  The `dict` keys are the visit IDs,
1266  and the values are `lsst.afw.image.ExposureF`
1267  The pre-loaded exposures for the current subregion.
1268  The variance plane contains weights, and not the variance
1269  """
1270  tempExpName = self.getTempExpDatasetName(self.warpType)
1271  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1272  subExposures = {}
1273  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1274  if isinstance(warpExpRef, DeferredDatasetHandle):
1275  exposure = warpExpRef.get(parameters={'bbox': bbox})
1276  else:
1277  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1278  visit = warpExpRef.dataId["visit"]
1279  if altMaskSpans is not None:
1280  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1281  imageScaler.scaleMaskedImage(exposure.maskedImage)
1282  # Note that the variance plane here is used to store weights, not the actual variance
1283  exposure.variance.array[:, :] = 0.
1284  # Set the weight of unmasked pixels to 1.
1285  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1286  # Set the image value of masked pixels to zero.
1287  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1288  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1289  subExposures[visit] = exposure
1290  return subExposures
1291 
1292  def selectCoaddPsf(self, templateCoadd, warpRefList):
1293  """Compute the PSF of the coadd from the exposures with the best seeing.
1294 
1295  Parameters
1296  ----------
1297  templateCoadd : `lsst.afw.image.ExposureF`
1298  The initial coadd exposure before accounting for DCR.
1299  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1300  `lsst.daf.persistence.ButlerDataRef`
1301  The data references to the input warped exposures.
1302 
1303  Returns
1304  -------
1305  psf : `lsst.meas.algorithms.CoaddPsf`
1306  The average PSF of the input exposures with the best seeing.
1307  """
1308  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1309  tempExpName = self.getTempExpDatasetName(self.warpType)
1310  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1311  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1312  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1313  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1314  psfSizes = np.zeros(len(ccds))
1315  ccdVisits = np.array(ccds["visit"])
1316  for warpExpRef in warpRefList:
1317  if isinstance(warpExpRef, DeferredDatasetHandle):
1318  # Gen 3 API
1319  psf = warpExpRef.get(component="psf")
1320  else:
1321  # Gen 2 API. Delete this when Gen 2 retired
1322  psf = warpExpRef.get(tempExpName).getPsf()
1323  visit = warpExpRef.dataId["visit"]
1324  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1325  psfSizes[ccdVisits == visit] = psfSize
1326  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1327  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1328  # of the mean or median FWHM of the input exposures.
1329  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1330  goodPsfs = psfSizes <= sizeThreshold
1331  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1332  self.config.coaddPsf.makeControl())
1333  return psf
lsst::afw::image
lsst.pipe.tasks.dcrAssembleCoadd.selectCoaddPsf
def selectCoaddPsf(self, templateCoadd, warpRefList)
Definition: dcrAssembleCoadd.py:1292
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig
Definition: assembleCoadd.py:1734
lsst.pipe.tasks.dcrAssembleCoadd.dcrResiduals
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
Definition: dcrAssembleCoadd.py:863
lsst.pipe.tasks.dcrAssembleCoadd.fillCoadd
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
Definition: dcrAssembleCoadd.py:1053
lsst::ip::diffim::dcrModel
lsst.pipe.tasks.dcrAssembleCoadd.dcrAssembleSubregion
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
Definition: dcrAssembleCoadd.py:791
lsst.pipe.tasks.dcrAssembleCoadd.calculateGain
def calculateGain(self, convergenceList, gainList)
Definition: dcrAssembleCoadd.py:1111
lsst.pipe.tasks.dcrAssembleCoadd.calculateSingleConvergence
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
Definition: dcrAssembleCoadd.py:993
lsst.pipe.tasks.dcrAssembleCoadd.DcrAssembleCoaddConnections
Definition: dcrAssembleCoadd.py:48
lsst.pipe.tasks.coaddBase.makeSkyInfo
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
lsst.pipe.tasks.dcrAssembleCoadd.calculateModelWeights
def calculateModelWeights(self, dcrModels, dcrBBox)
Definition: dcrAssembleCoadd.py:1192
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
Definition: assembleCoadd.py:1881
lsst.pipe.tasks.dcrAssembleCoadd.calculateConvergence
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Definition: dcrAssembleCoadd.py:950
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:714
lsst::meas::base
lsst.pipe.tasks.dcrAssembleCoadd.applyModelWeights
def applyModelWeights(self, modelImages, refImage, modelWeights)
Definition: dcrAssembleCoadd.py:1225
lsst.pipe.tasks.dcrAssembleCoadd.newModelFromResidual
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
Definition: dcrAssembleCoadd.py:895
lsst::afw::table
lsst::utils
lsst::pex::config
lsst::geom
lsst::ip::diffim::dcrModel::DcrModel
lsst.pipe.tasks.assembleCoadd.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:558
lsst::geom::Box2I
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe.tasks.dcrAssembleCoadd.stackCoadd
def stackCoadd(self, dcrCoadds)
Definition: dcrAssembleCoadd.py:1034
Extent< int, 2 >
lsst::coadd::utils
lsst.pipe.tasks.dcrAssembleCoadd.loadSubExposures
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
Definition: dcrAssembleCoadd.py:1245