lsst.pipe.tasks  21.0.0-21-g006371a9+f07e293ea4
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32 import lsst.meas.algorithms as measAlg
33 from lsst.meas.base import SingleFrameMeasurementTask
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 import lsst.utils as utils
37 from lsst.skymap import BaseSkyMap
38 from .assembleCoadd import (AssembleCoaddTask,
39  CompareWarpAssembleCoaddConfig,
40  CompareWarpAssembleCoaddTask)
41 from .coaddBase import makeSkyInfo
42 from .measurePsf import MeasurePsfTask
43 
44 __all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
45 
46 
47 class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
48  dimensions=("tract", "patch", "band", "skymap"),
49  defaultTemplates={"inputCoaddName": "deep",
50  "outputCoaddName": "dcr",
51  "warpType": "direct",
52  "warpTypeSuffix": "",
53  "fakesType": ""}):
54  inputWarps = pipeBase.connectionTypes.Input(
55  doc=("Input list of warps to be assembled i.e. stacked."
56  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
57  name="{inputCoaddName}Coadd_{warpType}Warp",
58  storageClass="ExposureF",
59  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
60  deferLoad=True,
61  multiple=True
62  )
63  skyMap = pipeBase.connectionTypes.Input(
64  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
65  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
66  storageClass="SkyMap",
67  dimensions=("skymap", ),
68  )
69  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
70  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
71  " BRIGHT_OBJECT."),
72  name="brightObjectMask",
73  storageClass="ObjectMaskCatalog",
74  dimensions=("tract", "patch", "skymap", "band"),
75  )
76  templateExposure = pipeBase.connectionTypes.Input(
77  doc="Input coadded exposure, produced by previous call to AssembleCoadd",
78  name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
79  storageClass="ExposureF",
80  dimensions=("tract", "patch", "skymap", "band"),
81  )
82  dcrCoadds = pipeBase.connectionTypes.Output(
83  doc="Output coadded exposure, produced by stacking input warps",
84  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
85  storageClass="ExposureF",
86  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
87  multiple=True,
88  )
89  dcrNImages = pipeBase.connectionTypes.Output(
90  doc="Output image of number of input images per pixel",
91  name="{outputCoaddName}Coadd_nImage",
92  storageClass="ImageU",
93  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
94  multiple=True,
95  )
96 
97  def __init__(self, *, config=None):
98  super().__init__(config=config)
99  if not config.doWrite:
100  self.outputs.remove("dcrCoadds")
101 
102 
103 class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
104  pipelineConnections=DcrAssembleCoaddConnections):
105  dcrNumSubfilters = pexConfig.Field(
106  dtype=int,
107  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
108  default=3,
109  )
110  maxNumIter = pexConfig.Field(
111  dtype=int,
112  optional=True,
113  doc="Maximum number of iterations of forward modeling.",
114  default=None,
115  )
116  minNumIter = pexConfig.Field(
117  dtype=int,
118  optional=True,
119  doc="Minimum number of iterations of forward modeling.",
120  default=None,
121  )
122  convergenceThreshold = pexConfig.Field(
123  dtype=float,
124  doc="Target relative change in convergence between iterations of forward modeling.",
125  default=0.001,
126  )
127  useConvergence = pexConfig.Field(
128  dtype=bool,
129  doc="Use convergence test as a forward modeling end condition?"
130  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
131  default=True,
132  )
133  baseGain = pexConfig.Field(
134  dtype=float,
135  optional=True,
136  doc="Relative weight to give the new solution vs. the last solution when updating the model."
137  "A value of 1.0 gives equal weight to both solutions."
138  "Small values imply slower convergence of the solution, but can "
139  "help prevent overshooting and failures in the fit."
140  "If ``baseGain`` is None, a conservative gain "
141  "will be calculated from the number of subfilters. ",
142  default=None,
143  )
144  useProgressiveGain = pexConfig.Field(
145  dtype=bool,
146  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
147  "When calculating the next gain, we use up to 5 previous gains and convergence values."
148  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
149  default=True,
150  )
151  doAirmassWeight = pexConfig.Field(
152  dtype=bool,
153  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
154  default=False,
155  )
156  modelWeightsWidth = pexConfig.Field(
157  dtype=float,
158  doc="Width of the region around detected sources to include in the DcrModel.",
159  default=3,
160  )
161  useModelWeights = pexConfig.Field(
162  dtype=bool,
163  doc="Width of the region around detected sources to include in the DcrModel.",
164  default=True,
165  )
166  splitSubfilters = pexConfig.Field(
167  dtype=bool,
168  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
169  "Instead of at the midpoint",
170  default=True,
171  )
172  splitThreshold = pexConfig.Field(
173  dtype=float,
174  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
175  "Set to 0 to always split the subfilters.",
176  default=0.1,
177  )
178  regularizeModelIterations = pexConfig.Field(
179  dtype=float,
180  doc="Maximum relative change of the model allowed between iterations."
181  "Set to zero to disable.",
182  default=2.,
183  )
184  regularizeModelFrequency = pexConfig.Field(
185  dtype=float,
186  doc="Maximum relative change of the model allowed between subfilters."
187  "Set to zero to disable.",
188  default=4.,
189  )
190  convergenceMaskPlanes = pexConfig.ListField(
191  dtype=str,
192  default=["DETECTED"],
193  doc="Mask planes to use to calculate convergence."
194  )
195  regularizationWidth = pexConfig.Field(
196  dtype=int,
197  default=2,
198  doc="Minimum radius of a region to include in regularization, in pixels."
199  )
200  imageInterpOrder = pexConfig.Field(
201  dtype=int,
202  doc="The order of the spline interpolation used to shift the image plane.",
203  default=3,
204  )
205  accelerateModel = pexConfig.Field(
206  dtype=float,
207  doc="Factor to amplify the differences between model planes by to speed convergence.",
208  default=3,
209  )
210  doCalculatePsf = pexConfig.Field(
211  dtype=bool,
212  doc="Set to detect stars and recalculate the PSF from the final coadd."
213  "Otherwise the PSF is estimated from a selection of the best input exposures",
214  default=False,
215  )
216  detectPsfSources = pexConfig.ConfigurableField(
217  target=measAlg.SourceDetectionTask,
218  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
219  )
220  measurePsfSources = pexConfig.ConfigurableField(
221  target=SingleFrameMeasurementTask,
222  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
223  )
224  measurePsf = pexConfig.ConfigurableField(
225  target=MeasurePsfTask,
226  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
227  )
228  effectiveWavelength = pexConfig.Field(
229  doc="Effective wavelength of the filter, in nm."
230  "Required if transmission curves aren't used."
231  "Support for using transmission curves is to be added in DM-13668.",
232  dtype=float,
233  )
234  bandwidth = pexConfig.Field(
235  doc="Bandwidth of the physical filter, in nm."
236  "Required if transmission curves aren't used."
237  "Support for using transmission curves is to be added in DM-13668.",
238  dtype=float,
239  )
240 
241  def setDefaults(self):
242  CompareWarpAssembleCoaddConfig.setDefaults(self)
243  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
244  self.doNImage = True
245  self.assembleStaticSkyModel.warpType = self.warpType
246  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
247  self.assembleStaticSkyModel.doNImage = False
248  self.assembleStaticSkyModel.doWrite = False
249  self.detectPsfSources.returnOriginalFootprints = False
250  self.detectPsfSources.thresholdPolarity = "positive"
251  # Only use bright sources for PSF measurement
252  self.detectPsfSources.thresholdValue = 50
253  self.detectPsfSources.nSigmaToGrow = 2
254  # A valid star for PSF measurement should at least fill 5x5 pixels
255  self.detectPsfSources.minPixels = 25
256  # Use the variance plane to calculate signal to noise
257  self.detectPsfSources.thresholdType = "pixel_stdev"
258  # The signal to noise limit is good enough, while the flux limit is set
259  # in dimensionless units and may not be appropriate for all data sets.
260  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
261 
262 
263 class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
264  """Assemble DCR coadded images from a set of warps.
265 
266  Attributes
267  ----------
268  bufferSize : `int`
269  The number of pixels to grow each subregion by to allow for DCR.
270 
271  Notes
272  -----
273  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
274  Warps (also called coadded temporary exposures), including the effects of
275  Differential Chromatic Refraction (DCR).
276  For full details of the mathematics and algorithm, please see
277  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
278 
279  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
280  each subfilter used in the iterative calculation.
281  It begins by dividing the bandpass-defining filter into N equal bandwidth
282  "subfilters", and divides the flux in each pixel from an initial coadd
283  equally into each as a "dcrModel". Because the airmass and parallactic
284  angle of each individual exposure is known, we can calculate the shift
285  relative to the center of the band in each subfilter due to DCR. For each
286  exposure we apply this shift as a linear transformation to the dcrModels
287  and stack the results to produce a DCR-matched exposure. The matched
288  exposures are subtracted from the input exposures to produce a set of
289  residual images, and these residuals are reverse shifted for each
290  exposures' subfilters and stacked. The shifted and stacked residuals are
291  added to the dcrModels to produce a new estimate of the flux in each pixel
292  within each subfilter. The dcrModels are solved for iteratively, which
293  continues until the solution from a new iteration improves by less than
294  a set percentage, or a maximum number of iterations is reached.
295  Two forms of regularization are employed to reduce unphysical results.
296  First, the new solution is averaged with the solution from the previous
297  iteration, which mitigates oscillating solutions where the model
298  overshoots with alternating very high and low values.
299  Second, a common degeneracy when the data have a limited range of airmass or
300  parallactic angle values is for one subfilter to be fit with very low or
301  negative values, while another subfilter is fit with very high values. This
302  typically appears in the form of holes next to sources in one subfilter,
303  and corresponding extended wings in another. Because each subfilter has
304  a narrow bandwidth we assume that physical sources that are above the noise
305  level will not vary in flux by more than a factor of `frequencyClampFactor`
306  between subfilters, and pixels that have flux deviations larger than that
307  factor will have the excess flux distributed evenly among all subfilters.
308  If `splitSubfilters` is set, then each subfilter will be further sub-
309  divided during the forward modeling step (only). This approximates using
310  a higher number of subfilters that may be necessary for high airmass
311  observations, but does not increase the number of free parameters in the
312  fit. This is needed when there are high airmass observations which would
313  otherwise have significant DCR even within a subfilter. Because calculating
314  the shifted images takes most of the time, splitting the subfilters is
315  turned off by way of the `splitThreshold` option for low-airmass
316  observations that do not suffer from DCR within a subfilter.
317  """
318 
319  ConfigClass = DcrAssembleCoaddConfig
320  _DefaultName = "dcrAssembleCoadd"
321 
322  def __init__(self, *args, **kwargs):
323  super().__init__(*args, **kwargs)
324  if self.config.doCalculatePsf:
325  self.schema = afwTable.SourceTable.makeMinimalSchema()
326  self.makeSubtask("detectPsfSources", schema=self.schema)
327  self.makeSubtask("measurePsfSources", schema=self.schema)
328  self.makeSubtask("measurePsf", schema=self.schema)
329 
330  @utils.inheritDoc(pipeBase.PipelineTask)
331  def runQuantum(self, butlerQC, inputRefs, outputRefs):
332  # Docstring to be formatted with info from PipelineTask.runQuantum
333  """
334  Notes
335  -----
336  Assemble a coadd from a set of Warps.
337 
338  PipelineTask (Gen3) entry point to Coadd a set of Warps.
339  Analogous to `runDataRef`, it prepares all the data products to be
340  passed to `run`, and processes the results before returning a struct
341  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
342  Therefore, its inputs are accessed subregion by subregion
343  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
344  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
345  correspond to an update in `runDataRef` while both entry points
346  are used.
347  """
348  inputData = butlerQC.get(inputRefs)
349 
350  # Construct skyInfo expected by run
351  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
352  skyMap = inputData["skyMap"]
353  outputDataId = butlerQC.quantum.dataId
354 
355  inputData['skyInfo'] = makeSkyInfo(skyMap,
356  tractId=outputDataId['tract'],
357  patchId=outputDataId['patch'])
358 
359  # Construct list of input Deferred Datasets
360  # These quack a bit like like Gen2 DataRefs
361  warpRefList = inputData['inputWarps']
362  # Perform same middle steps as `runDataRef` does
363  inputs = self.prepareInputs(warpRefList)
364  self.log.info("Found %d %s", len(inputs.tempExpRefList),
365  self.getTempExpDatasetName(self.warpType))
366  if len(inputs.tempExpRefList) == 0:
367  self.log.warn("No coadd temporary exposures found")
368  return
369 
370  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
371  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
372  inputs.weightList, supplementaryData=supplementaryData)
373 
374  inputData.setdefault('brightObjectMask', None)
375  for subfilter in range(self.config.dcrNumSubfilters):
376  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
377  retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
378  self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
379 
380  if self.config.doWrite:
381  butlerQC.put(retStruct, outputRefs)
382  return retStruct
383 
384  @pipeBase.timeMethod
385  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
386  """Assemble a coadd from a set of warps.
387 
388  Coadd a set of Warps. Compute weights to be applied to each Warp and
389  find scalings to match the photometric zeropoint to a reference Warp.
390  Assemble the Warps using run method.
391  Forward model chromatic effects across multiple subfilters,
392  and subtract from the input Warps to build sets of residuals.
393  Use the residuals to construct a new ``DcrModel`` for each subfilter,
394  and iterate until the model converges.
395  Interpolate over NaNs and optionally write the coadd to disk.
396  Return the coadded exposure.
397 
398  Parameters
399  ----------
400  dataRef : `lsst.daf.persistence.ButlerDataRef`
401  Data reference defining the patch for coaddition and the
402  reference Warp
403  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
404  List of data references to warps. Data to be coadded will be
405  selected from this list based on overlap with the patch defined by
406  the data reference.
407 
408  Returns
409  -------
410  results : `lsst.pipe.base.Struct`
411  The Struct contains the following fields:
412 
413  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
414  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
415  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
416  - ``dcrNImages``: `list` of exposure count images for each subfilter
417  """
418  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
419  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
420 
421  skyInfo = self.getSkyInfo(dataRef)
422  if warpRefList is None:
423  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
424  if len(calExpRefList) == 0:
425  self.log.warn("No exposures to coadd")
426  return
427  self.log.info("Coadding %d exposures", len(calExpRefList))
428 
429  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
430 
431  inputData = self.prepareInputs(warpRefList)
432  self.log.info("Found %d %s", len(inputData.tempExpRefList),
433  self.getTempExpDatasetName(self.warpType))
434  if len(inputData.tempExpRefList) == 0:
435  self.log.warn("No coadd temporary exposures found")
436  return
437 
438  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
439 
440  results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
441  inputData.weightList, supplementaryData=supplementaryData)
442  if results is None:
443  self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
444  skyInfo.patchInfo.getIndex())
445  return
446 
447  if self.config.doCalculatePsf:
448  self.measureCoaddPsf(results.coaddExposure)
449  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
450  for subfilter in range(self.config.dcrNumSubfilters):
451  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
452  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
453  self.processResults(results.dcrCoadds[subfilter],
454  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
455  if self.config.doWrite:
456  self.log.info("Persisting dcrCoadd")
457  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
458  numSubfilters=self.config.dcrNumSubfilters)
459  if self.config.doNImage and results.dcrNImages is not None:
460  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
461  numSubfilters=self.config.dcrNumSubfilters)
462 
463  return results
464 
465  @utils.inheritDoc(AssembleCoaddTask)
466  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
467  """Load the previously-generated template coadd.
468 
469  This can be removed entirely once we no longer support the Gen 2 butler.
470 
471  Returns
472  -------
473  templateCoadd : `lsst.pipe.base.Struct`
474  Result struct with components:
475 
476  - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
477  """
478  templateCoadd = butlerQC.get(inputRefs.templateExposure)
479 
480  return pipeBase.Struct(templateCoadd=templateCoadd)
481 
482  def measureCoaddPsf(self, coaddExposure):
483  """Detect sources on the coadd exposure and measure the final PSF.
484 
485  Parameters
486  ----------
487  coaddExposure : `lsst.afw.image.Exposure`
488  The final coadded exposure.
489  """
490  table = afwTable.SourceTable.make(self.schema)
491  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
492  coaddSources = detResults.sources
493  self.measurePsfSources.run(
494  measCat=coaddSources,
495  exposure=coaddExposure
496  )
497  # Measure the PSF on the stacked subfilter coadds if possible.
498  # We should already have a decent estimate of the coadd PSF, however,
499  # so in case of any errors simply log them as a warning and use the
500  # default PSF.
501  try:
502  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
503  except Exception as e:
504  self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
505  else:
506  coaddExposure.setPsf(psfResults.psf)
507 
508  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
509  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
510 
511  Sets the property ``bufferSize``.
512 
513  Parameters
514  ----------
515  templateCoadd : `lsst.afw.image.ExposureF`
516  The initial coadd exposure before accounting for DCR.
517  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
518  `lsst.daf.persistence.ButlerDataRef`
519  The data references to the input warped exposures.
520  weightList : `list` of `float`
521  The weight to give each input exposure in the coadd
522  Will be modified in place if ``doAirmassWeight`` is set.
523 
524  Returns
525  -------
526  dcrModels : `lsst.pipe.tasks.DcrModel`
527  Best fit model of the true sky after correcting chromatic effects.
528 
529  Raises
530  ------
531  NotImplementedError
532  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
533  """
534  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
535  filterInfo = templateCoadd.getFilter()
536  tempExpName = self.getTempExpDatasetName(self.warpType)
537  dcrShifts = []
538  airmassDict = {}
539  angleDict = {}
540  psfSizeDict = {}
541  for visitNum, warpExpRef in enumerate(warpRefList):
542  if isinstance(warpExpRef, DeferredDatasetHandle):
543  # Gen 3 API
544  visitInfo = warpExpRef.get(component="visitInfo")
545  psf = warpExpRef.get(component="psf")
546  else:
547  # Gen 2 API. Delete this when Gen 2 retired
548  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
549  psf = warpExpRef.get(tempExpName).getPsf()
550  visit = warpExpRef.dataId["visit"]
551  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
552  airmass = visitInfo.getBoresightAirmass()
553  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
554  airmassDict[visit] = airmass
555  angleDict[visit] = parallacticAngle
556  psfSizeDict[visit] = psfSize
557  if self.config.doAirmassWeight:
558  weightList[visitNum] *= airmass
559  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
560  self.config.effectiveWavelength,
561  self.config.bandwidth,
562  self.config.dcrNumSubfilters))))
563  self.log.info("Selected airmasses:\n%s", airmassDict)
564  self.log.info("Selected parallactic angles:\n%s", angleDict)
565  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
566  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
567  try:
568  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
569  except Exception as e:
570  self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e)
571  else:
572  psf = templateCoadd.getPsf()
573  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
574  self.config.dcrNumSubfilters,
575  effectiveWavelength=self.config.effectiveWavelength,
576  bandwidth=self.config.bandwidth,
577  filterInfo=filterInfo,
578  psf=psf)
579  return dcrModels
580 
581  @pipeBase.timeMethod
582  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
583  supplementaryData=None):
584  """Assemble the coadd.
585 
586  Requires additional inputs Struct ``supplementaryData`` to contain a
587  ``templateCoadd`` that serves as the model of the static sky.
588 
589  Find artifacts and apply them to the warps' masks creating a list of
590  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
591  Then pass these alternative masks to the base class's assemble method.
592 
593  Divide the ``templateCoadd`` evenly between each subfilter of a
594  ``DcrModel`` as the starting best estimate of the true wavelength-
595  dependent sky. Forward model the ``DcrModel`` using the known
596  chromatic effects in each subfilter and calculate a convergence metric
597  based on how well the modeled template matches the input warps. If
598  the convergence has not yet reached the desired threshold, then shift
599  and stack the residual images to build a new ``DcrModel``. Apply
600  conditioning to prevent oscillating solutions between iterations or
601  between subfilters.
602 
603  Once the ``DcrModel`` reaches convergence or the maximum number of
604  iterations has been reached, fill the metadata for each subfilter
605  image and make them proper ``coaddExposure``s.
606 
607  Parameters
608  ----------
609  skyInfo : `lsst.pipe.base.Struct`
610  Patch geometry information, from getSkyInfo
611  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
612  `lsst.daf.persistence.ButlerDataRef`
613  The data references to the input warped exposures.
614  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
615  The image scalars correct for the zero point of the exposures.
616  weightList : `list` of `float`
617  The weight to give each input exposure in the coadd
618  supplementaryData : `lsst.pipe.base.Struct`
619  Result struct returned by ``makeSupplementaryData`` with components:
620 
621  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
622 
623  Returns
624  -------
625  result : `lsst.pipe.base.Struct`
626  Result struct with components:
627 
628  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
629  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
630  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
631  - ``dcrNImages``: `list` of exposure count images for each subfilter
632  """
633  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
634  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
635  templateCoadd = supplementaryData.templateCoadd
636  baseMask = templateCoadd.mask.clone()
637  # The variance plane is for each subfilter
638  # and should be proportionately lower than the full-band image
639  baseVariance = templateCoadd.variance.clone()
640  baseVariance /= self.config.dcrNumSubfilters
641  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
642  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
643  templateCoadd.setMask(baseMask)
644  badMaskPlanes = self.config.badMaskPlanes[:]
645  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
646  # This is because pixels in observations that are significantly affect by DCR
647  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
648  # but those are necessary to constrain the DCR model.
649  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
650 
651  stats = self.prepareStats(mask=badPixelMask)
652  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
653  if self.config.doNImage:
654  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
655  spanSetMaskList, stats.ctrl)
656  nImage = afwImage.ImageU(skyInfo.bbox)
657  # Note that this nImage will be a factor of dcrNumSubfilters higher than
658  # the nImage returned by assembleCoadd for most pixels. This is because each
659  # subfilter may have a different nImage, and fractional values are not allowed.
660  for dcrNImage in dcrNImages:
661  nImage += dcrNImage
662  else:
663  dcrNImages = None
664 
665  subregionSize = geom.Extent2I(*self.config.subregionSize)
666  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
667  * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
668  subIter = 0
669  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
670  modelIter = 0
671  subIter += 1
672  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
673  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
674  dcrBBox = geom.Box2I(subBBox)
675  dcrBBox.grow(self.bufferSize)
676  dcrBBox.clip(dcrModels.bbox)
677  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
678  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
679  imageScalerList, spanSetMaskList)
680  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
681  warpRefList, weightList, stats.ctrl)
682  self.log.info("Initial convergence : %s", convergenceMetric)
683  convergenceList = [convergenceMetric]
684  gainList = []
685  convergenceCheck = 1.
686  refImage = templateCoadd.image
687  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
688  gain = self.calculateGain(convergenceList, gainList)
689  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
690  stats.ctrl, convergenceMetric, gain,
691  modelWeights, refImage, dcrWeights)
692  if self.config.useConvergence:
693  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
694  warpRefList, weightList, stats.ctrl)
695  if convergenceMetric == 0:
696  self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
697  "most likely due to there being no valid data in the region.",
698  skyInfo.patchInfo.getIndex(), subIter)
699  break
700  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
701  if (convergenceCheck < 0) & (modelIter > minNumIter):
702  self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
703  "iterations or desired convergence improvement of %s."
704  " Divergence: %s",
705  skyInfo.patchInfo.getIndex(), subIter,
706  self.config.convergenceThreshold, convergenceCheck)
707  break
708  convergenceList.append(convergenceMetric)
709  if modelIter > maxNumIter:
710  if self.config.useConvergence:
711  self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
712  "before reaching desired convergence improvement of %s."
713  " Final convergence improvement: %s",
714  skyInfo.patchInfo.getIndex(), subIter,
715  self.config.convergenceThreshold, convergenceCheck)
716  break
717 
718  if self.config.useConvergence:
719  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
720  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
721  modelIter += 1
722  else:
723  if self.config.useConvergence:
724  self.log.info("Coadd patch %s subregion %s finished with "
725  "convergence metric %s after %s iterations",
726  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
727  else:
728  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
729  skyInfo.patchInfo.getIndex(), subIter, modelIter)
730  if self.config.useConvergence and convergenceMetric > 0:
731  self.log.info("Final convergence improvement was %.4f%% overall",
732  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
733 
734  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
735  calibration=self.scaleZeroPoint.getPhotoCalib(),
736  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
737  mask=baseMask,
738  variance=baseVariance)
739  coaddExposure = self.stackCoadd(dcrCoadds)
740  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
741  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
742 
743  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
744  """Calculate the number of exposures contributing to each subfilter.
745 
746  Parameters
747  ----------
748  dcrModels : `lsst.pipe.tasks.DcrModel`
749  Best fit model of the true sky after correcting chromatic effects.
750  bbox : `lsst.geom.box.Box2I`
751  Bounding box of the patch to coadd.
752  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
753  `lsst.daf.persistence.ButlerDataRef`
754  The data references to the input warped exposures.
755  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
756  Each element of the `dict` contains the new mask plane name
757  (e.g. "CLIPPED and/or "NO_DATA") as the key,
758  and the list of SpanSets to apply to the mask.
759  statsCtrl : `lsst.afw.math.StatisticsControl`
760  Statistics control object for coadd
761 
762  Returns
763  -------
764  dcrNImages : `list` of `lsst.afw.image.ImageU`
765  List of exposure count images for each subfilter
766  dcrWeights : `list` of `lsst.afw.image.ImageF`
767  Per-pixel weights for each subfilter.
768  Equal to 1/(number of unmasked images contributing to each pixel).
769  """
770  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
771  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
772  tempExpName = self.getTempExpDatasetName(self.warpType)
773  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
774  if isinstance(warpExpRef, DeferredDatasetHandle):
775  # Gen 3 API
776  exposure = warpExpRef.get(parameters={'bbox': bbox})
777  else:
778  # Gen 2 API. Delete this when Gen 2 retired
779  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
780  visitInfo = exposure.getInfo().getVisitInfo()
781  wcs = exposure.getInfo().getWcs()
782  mask = exposure.mask
783  if altMaskSpans is not None:
784  self.applyAltMaskPlanes(mask, altMaskSpans)
785  weightImage = np.zeros_like(exposure.image.array)
786  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
787  # The weights must be shifted in exactly the same way as the residuals,
788  # because they will be used as the denominator in the weighted average of residuals.
789  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
790  dcrModels.effectiveWavelength, dcrModels.bandwidth)
791  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
792  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
793  dcrWeight.array += shiftedWeights
794  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
795  weightsThreshold = 1.
796  goodPix = dcrWeights[0].array > weightsThreshold
797  for weights in dcrWeights[1:]:
798  goodPix = (weights.array > weightsThreshold) & goodPix
799  for subfilter in range(self.config.dcrNumSubfilters):
800  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
801  dcrWeights[subfilter].array[~goodPix] = 0.
802  dcrNImages[subfilter].array[~goodPix] = 0
803  return (dcrNImages, dcrWeights)
804 
805  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
806  statsCtrl, convergenceMetric,
807  gain, modelWeights, refImage, dcrWeights):
808  """Assemble the DCR coadd for a sub-region.
809 
810  Build a DCR-matched template for each input exposure, then shift the
811  residuals according to the DCR in each subfilter.
812  Stack the shifted residuals and apply them as a correction to the
813  solution from the previous iteration.
814  Restrict the new model solutions from varying by more than a factor of
815  `modelClampFactor` from the last solution, and additionally restrict the
816  individual subfilter models from varying by more than a factor of
817  `frequencyClampFactor` from their average.
818  Finally, mitigate potentially oscillating solutions by averaging the new
819  solution with the solution from the previous iteration, weighted by
820  their convergence metric.
821 
822  Parameters
823  ----------
824  dcrModels : `lsst.pipe.tasks.DcrModel`
825  Best fit model of the true sky after correcting chromatic effects.
826  subExposures : `dict` of `lsst.afw.image.ExposureF`
827  The pre-loaded exposures for the current subregion.
828  bbox : `lsst.geom.box.Box2I`
829  Bounding box of the subregion to coadd.
830  dcrBBox : `lsst.geom.box.Box2I`
831  Sub-region of the coadd which includes a buffer to allow for DCR.
832  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
833  `lsst.daf.persistence.ButlerDataRef`
834  The data references to the input warped exposures.
835  statsCtrl : `lsst.afw.math.StatisticsControl`
836  Statistics control object for coadd
837  convergenceMetric : `float`
838  Quality of fit metric for the matched templates of the input images.
839  gain : `float`, optional
840  Relative weight to give the new solution when updating the model.
841  modelWeights : `numpy.ndarray` or `float`
842  A 2D array of weight values that tapers smoothly to zero away from detected sources.
843  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
844  refImage : `lsst.afw.image.Image`
845  A reference image used to supply the default pixel values.
846  dcrWeights : `list` of `lsst.afw.image.Image`
847  Per-pixel weights for each subfilter.
848  Equal to 1/(number of unmasked images contributing to each pixel).
849  """
850  residualGeneratorList = []
851 
852  for warpExpRef in warpRefList:
853  visit = warpExpRef.dataId["visit"]
854  exposure = subExposures[visit]
855  visitInfo = exposure.getInfo().getVisitInfo()
856  wcs = exposure.getInfo().getWcs()
857  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
858  order=self.config.imageInterpOrder,
859  splitSubfilters=self.config.splitSubfilters,
860  splitThreshold=self.config.splitThreshold,
861  amplifyModel=self.config.accelerateModel)
862  residual = exposure.image.array - templateImage.array
863  # Note that the variance plane here is used to store weights, not the actual variance
864  residual *= exposure.variance.array
865  # The residuals are stored as a list of generators.
866  # This allows the residual for a given subfilter and exposure to be created
867  # on the fly, instead of needing to store them all in memory.
868  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
869  dcrModels.effectiveWavelength,
870  dcrModels.bandwidth))
871 
872  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
873  gain=gain,
874  modelWeights=modelWeights,
875  refImage=refImage,
876  dcrWeights=dcrWeights)
877  dcrModels.assign(dcrSubModelOut, bbox)
878 
879  def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
880  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
881 
882  Parameters
883  ----------
884  residual : `numpy.ndarray`
885  The residual masked image for one exposure,
886  after subtracting the matched template
887  visitInfo : `lsst.afw.image.VisitInfo`
888  Metadata for the exposure.
889  wcs : `lsst.afw.geom.SkyWcs`
890  Coordinate system definition (wcs) for the exposure.
891  filterInfo : `lsst.afw.image.Filter`
892  The filter definition, set in the current instruments' obs package.
893  Note: this object will be changed in DM-21333.
894 
895  Yields
896  ------
897  residualImage : `numpy.ndarray`
898  The residual image for the next subfilter, shifted for DCR.
899  """
900  # Pre-calculate the spline-filtered residual image, so that step can be
901  # skipped in the shift calculation in `applyDcr`.
902  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
903  # Note that `splitSubfilters` is always turned off in the reverse direction.
904  # This option introduces additional blurring if applied to the residuals.
905  dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
906  splitSubfilters=False)
907  for dcr in dcrShift:
908  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
909  doPrefilter=False, order=self.config.imageInterpOrder)
910 
911  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
912  gain, modelWeights, refImage, dcrWeights):
913  """Calculate a new DcrModel from a set of image residuals.
914 
915  Parameters
916  ----------
917  dcrModels : `lsst.pipe.tasks.DcrModel`
918  Current model of the true sky after correcting chromatic effects.
919  residualGeneratorList : `generator` of `numpy.ndarray`
920  The residual image for the next subfilter, shifted for DCR.
921  dcrBBox : `lsst.geom.box.Box2I`
922  Sub-region of the coadd which includes a buffer to allow for DCR.
923  statsCtrl : `lsst.afw.math.StatisticsControl`
924  Statistics control object for coadd
925  gain : `float`
926  Relative weight to give the new solution when updating the model.
927  modelWeights : `numpy.ndarray` or `float`
928  A 2D array of weight values that tapers smoothly to zero away from detected sources.
929  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
930  refImage : `lsst.afw.image.Image`
931  A reference image used to supply the default pixel values.
932  dcrWeights : `list` of `lsst.afw.image.Image`
933  Per-pixel weights for each subfilter.
934  Equal to 1/(number of unmasked images contributing to each pixel).
935 
936  Returns
937  -------
938  dcrModel : `lsst.pipe.tasks.DcrModel`
939  New model of the true sky after correcting chromatic effects.
940  """
941  newModelImages = []
942  for subfilter, model in enumerate(dcrModels):
943  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
944  residual = np.sum(residualsList, axis=0)
945  residual *= dcrWeights[subfilter][dcrBBox].array
946  # `MaskedImage`s only support in-place addition, so rename for readability
947  newModel = model[dcrBBox].clone()
948  newModel.array += residual
949  # Catch any invalid values
950  badPixels = ~np.isfinite(newModel.array)
951  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
952  if self.config.regularizeModelIterations > 0:
953  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
954  self.config.regularizeModelIterations,
955  self.config.regularizationWidth)
956  newModelImages.append(newModel)
957  if self.config.regularizeModelFrequency > 0:
958  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
959  self.config.regularizeModelFrequency,
960  self.config.regularizationWidth)
961  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
962  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
963  return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
964  dcrModels.bandwidth, dcrModels.psf,
965  dcrModels.mask, dcrModels.variance)
966 
967  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
968  """Calculate a quality of fit metric for the matched templates.
969 
970  Parameters
971  ----------
972  dcrModels : `lsst.pipe.tasks.DcrModel`
973  Best fit model of the true sky after correcting chromatic effects.
974  subExposures : `dict` of `lsst.afw.image.ExposureF`
975  The pre-loaded exposures for the current subregion.
976  bbox : `lsst.geom.box.Box2I`
977  Sub-region to coadd
978  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
979  `lsst.daf.persistence.ButlerDataRef`
980  The data references to the input warped exposures.
981  weightList : `list` of `float`
982  The weight to give each input exposure in the coadd
983  statsCtrl : `lsst.afw.math.StatisticsControl`
984  Statistics control object for coadd
985 
986  Returns
987  -------
988  convergenceMetric : `float`
989  Quality of fit metric for all input exposures, within the sub-region
990  """
991  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
992  nSigma = 3.
993  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
994  bufferSize=self.bufferSize)
995  if np.max(significanceImage) == 0:
996  significanceImage += 1.
997  weight = 0
998  metric = 0.
999  metricList = {}
1000  for warpExpRef, expWeight in zip(warpRefList, weightList):
1001  visit = warpExpRef.dataId["visit"]
1002  exposure = subExposures[visit][bbox]
1003  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
1004  metric += singleMetric
1005  metricList[visit] = singleMetric
1006  weight += 1.
1007  self.log.info("Individual metrics:\n%s", metricList)
1008  return 1.0 if weight == 0.0 else metric/weight
1009 
1010  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
1011  """Calculate a quality of fit metric for a single matched template.
1012 
1013  Parameters
1014  ----------
1015  dcrModels : `lsst.pipe.tasks.DcrModel`
1016  Best fit model of the true sky after correcting chromatic effects.
1017  exposure : `lsst.afw.image.ExposureF`
1018  The input warped exposure to evaluate.
1019  significanceImage : `numpy.ndarray`
1020  Array of weights for each pixel corresponding to its significance
1021  for the convergence calculation.
1022  statsCtrl : `lsst.afw.math.StatisticsControl`
1023  Statistics control object for coadd
1024 
1025  Returns
1026  -------
1027  convergenceMetric : `float`
1028  Quality of fit metric for one exposure, within the sub-region.
1029  """
1030  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1031  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1032  order=self.config.imageInterpOrder,
1033  splitSubfilters=self.config.splitSubfilters,
1034  splitThreshold=self.config.splitThreshold,
1035  amplifyModel=self.config.accelerateModel)
1036  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1037  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1038 
1039  finitePixels = np.isfinite(diffVals)
1040  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1041  convergeMaskPixels = exposure.mask.array & convergeMask > 0
1042  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1043  if np.sum(usePixels) == 0:
1044  metric = 0.
1045  else:
1046  diffUse = diffVals[usePixels]
1047  refUse = refVals[usePixels]
1048  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1049  return metric
1050 
1051  def stackCoadd(self, dcrCoadds):
1052  """Add a list of sub-band coadds together.
1053 
1054  Parameters
1055  ----------
1056  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1057  A list of coadd exposures, each exposure containing
1058  the model for one subfilter.
1059 
1060  Returns
1061  -------
1062  coaddExposure : `lsst.afw.image.ExposureF`
1063  A single coadd exposure that is the sum of the sub-bands.
1064  """
1065  coaddExposure = dcrCoadds[0].clone()
1066  for coadd in dcrCoadds[1:]:
1067  coaddExposure.maskedImage += coadd.maskedImage
1068  return coaddExposure
1069 
1070  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1071  mask=None, variance=None):
1072  """Create a list of coadd exposures from a list of masked images.
1073 
1074  Parameters
1075  ----------
1076  dcrModels : `lsst.pipe.tasks.DcrModel`
1077  Best fit model of the true sky after correcting chromatic effects.
1078  skyInfo : `lsst.pipe.base.Struct`
1079  Patch geometry information, from getSkyInfo
1080  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1081  `lsst.daf.persistence.ButlerDataRef`
1082  The data references to the input warped exposures.
1083  weightList : `list` of `float`
1084  The weight to give each input exposure in the coadd
1085  calibration : `lsst.afw.Image.PhotoCalib`, optional
1086  Scale factor to set the photometric calibration of an exposure.
1087  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1088  A record of the observations that are included in the coadd.
1089  mask : `lsst.afw.image.Mask`, optional
1090  Optional mask to override the values in the final coadd.
1091  variance : `lsst.afw.image.Image`, optional
1092  Optional variance plane to override the values in the final coadd.
1093 
1094  Returns
1095  -------
1096  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1097  A list of coadd exposures, each exposure containing
1098  the model for one subfilter.
1099  """
1100  dcrCoadds = []
1101  refModel = dcrModels.getReferenceImage()
1102  for model in dcrModels:
1103  if self.config.accelerateModel > 1:
1104  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1105  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1106  if calibration is not None:
1107  coaddExposure.setPhotoCalib(calibration)
1108  if coaddInputs is not None:
1109  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1110  # Set the metadata for the coadd, including PSF and aperture corrections.
1111  self.assembleMetadata(coaddExposure, warpRefList, weightList)
1112  # Overwrite the PSF
1113  coaddExposure.setPsf(dcrModels.psf)
1114  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1115  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1116  maskedImage.image = model
1117  maskedImage.mask = dcrModels.mask
1118  maskedImage.variance = dcrModels.variance
1119  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1120  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1121  if mask is not None:
1122  coaddExposure.setMask(mask)
1123  if variance is not None:
1124  coaddExposure.setVariance(variance)
1125  dcrCoadds.append(coaddExposure)
1126  return dcrCoadds
1127 
1128  def calculateGain(self, convergenceList, gainList):
1129  """Calculate the gain to use for the current iteration.
1130 
1131  After calculating a new DcrModel, each value is averaged with the
1132  value in the corresponding pixel from the previous iteration. This
1133  reduces oscillating solutions that iterative techniques are plagued by,
1134  and speeds convergence. By far the biggest changes to the model
1135  happen in the first couple iterations, so we can also use a more
1136  aggressive gain later when the model is changing slowly.
1137 
1138  Parameters
1139  ----------
1140  convergenceList : `list` of `float`
1141  The quality of fit metric from each previous iteration.
1142  gainList : `list` of `float`
1143  The gains used in each previous iteration: appended with the new
1144  gain value.
1145  Gains are numbers between ``self.config.baseGain`` and 1.
1146 
1147  Returns
1148  -------
1149  gain : `float`
1150  Relative weight to give the new solution when updating the model.
1151  A value of 1.0 gives equal weight to both solutions.
1152 
1153  Raises
1154  ------
1155  ValueError
1156  If ``len(convergenceList) != len(gainList)+1``.
1157  """
1158  nIter = len(convergenceList)
1159  if nIter != len(gainList) + 1:
1160  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1161  % (len(convergenceList), len(gainList)))
1162 
1163  if self.config.baseGain is None:
1164  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1165  # The more subfilters being modeled, the lower the gain should be.
1166  baseGain = 1./(self.config.dcrNumSubfilters - 1)
1167  else:
1168  baseGain = self.config.baseGain
1169 
1170  if self.config.useProgressiveGain and nIter > 2:
1171  # To calculate the best gain to use, compare the past gains that have been used
1172  # with the resulting convergences to estimate the best gain to use.
1173  # Algorithmically, this is a Kalman filter.
1174  # If forward modeling proceeds perfectly, the convergence metric should
1175  # asymptotically approach a final value.
1176  # We can estimate that value from the measured changes in convergence
1177  # weighted by the gains used in each previous iteration.
1178  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1179  for i in range(nIter - 1)]
1180  # The convergence metric is strictly positive, so if the estimated final convergence is
1181  # less than zero, force it to zero.
1182  estFinalConv = np.array(estFinalConv)
1183  estFinalConv[estFinalConv < 0] = 0
1184  # Because the estimate may slowly change over time, only use the most recent measurements.
1185  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1186  lastGain = gainList[-1]
1187  lastConv = convergenceList[-2]
1188  newConv = convergenceList[-1]
1189  # The predicted convergence is the value we would get if the new model calculated
1190  # in the previous iteration was perfect. Recall that the updated model that is
1191  # actually used is the gain-weighted average of the new and old model,
1192  # so the convergence would be similarly weighted.
1193  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1194  # If the measured and predicted convergence are very close, that indicates
1195  # that our forward model is accurate and we can use a more aggressive gain
1196  # If the measured convergence is significantly worse (or better!) than predicted,
1197  # that indicates that the model is not converging as expected and
1198  # we should use a more conservative gain.
1199  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1200  newGain = 1 - abs(delta)
1201  # Average the gains to prevent oscillating solutions.
1202  newGain = (newGain + lastGain)/2.
1203  gain = max(baseGain, newGain)
1204  else:
1205  gain = baseGain
1206  gainList.append(gain)
1207  return gain
1208 
1209  def calculateModelWeights(self, dcrModels, dcrBBox):
1210  """Build an array that smoothly tapers to 0 away from detected sources.
1211 
1212  Parameters
1213  ----------
1214  dcrModels : `lsst.pipe.tasks.DcrModel`
1215  Best fit model of the true sky after correcting chromatic effects.
1216  dcrBBox : `lsst.geom.box.Box2I`
1217  Sub-region of the coadd which includes a buffer to allow for DCR.
1218 
1219  Returns
1220  -------
1221  weights : `numpy.ndarray` or `float`
1222  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1223  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1224 
1225  Raises
1226  ------
1227  ValueError
1228  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1229  """
1230  if not self.config.useModelWeights:
1231  return 1.0
1232  if self.config.modelWeightsWidth < 0:
1233  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1234  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1235  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1236  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1237  weights[convergeMaskPixels] = 1.
1238  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1239  weights /= np.max(weights)
1240  return weights
1241 
1242  def applyModelWeights(self, modelImages, refImage, modelWeights):
1243  """Smoothly replace model pixel values with those from a
1244  reference at locations away from detected sources.
1245 
1246  Parameters
1247  ----------
1248  modelImages : `list` of `lsst.afw.image.Image`
1249  The new DCR model images from the current iteration.
1250  The values will be modified in place.
1251  refImage : `lsst.afw.image.MaskedImage`
1252  A reference image used to supply the default pixel values.
1253  modelWeights : `numpy.ndarray` or `float`
1254  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1255  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1256  """
1257  if self.config.useModelWeights:
1258  for model in modelImages:
1259  model.array *= modelWeights
1260  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1261 
1262  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1263  """Pre-load sub-regions of a list of exposures.
1264 
1265  Parameters
1266  ----------
1267  bbox : `lsst.geom.box.Box2I`
1268  Sub-region to coadd
1269  statsCtrl : `lsst.afw.math.StatisticsControl`
1270  Statistics control object for coadd
1271  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1272  `lsst.daf.persistence.ButlerDataRef`
1273  The data references to the input warped exposures.
1274  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1275  The image scalars correct for the zero point of the exposures.
1276  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1277  Each element is dict with keys = mask plane name to add the spans to
1278 
1279  Returns
1280  -------
1281  subExposures : `dict`
1282  The `dict` keys are the visit IDs,
1283  and the values are `lsst.afw.image.ExposureF`
1284  The pre-loaded exposures for the current subregion.
1285  The variance plane contains weights, and not the variance
1286  """
1287  tempExpName = self.getTempExpDatasetName(self.warpType)
1288  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1289  subExposures = {}
1290  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1291  if isinstance(warpExpRef, DeferredDatasetHandle):
1292  exposure = warpExpRef.get(parameters={'bbox': bbox})
1293  else:
1294  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1295  visit = warpExpRef.dataId["visit"]
1296  if altMaskSpans is not None:
1297  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1298  imageScaler.scaleMaskedImage(exposure.maskedImage)
1299  # Note that the variance plane here is used to store weights, not the actual variance
1300  exposure.variance.array[:, :] = 0.
1301  # Set the weight of unmasked pixels to 1.
1302  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1303  # Set the image value of masked pixels to zero.
1304  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1305  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1306  subExposures[visit] = exposure
1307  return subExposures
1308 
1309  def selectCoaddPsf(self, templateCoadd, warpRefList):
1310  """Compute the PSF of the coadd from the exposures with the best seeing.
1311 
1312  Parameters
1313  ----------
1314  templateCoadd : `lsst.afw.image.ExposureF`
1315  The initial coadd exposure before accounting for DCR.
1316  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1317  `lsst.daf.persistence.ButlerDataRef`
1318  The data references to the input warped exposures.
1319 
1320  Returns
1321  -------
1322  psf : `lsst.meas.algorithms.CoaddPsf`
1323  The average PSF of the input exposures with the best seeing.
1324  """
1325  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1326  tempExpName = self.getTempExpDatasetName(self.warpType)
1327  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1328  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1329  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1330  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1331  psfSizes = np.zeros(len(ccds))
1332  ccdVisits = np.array(ccds["visit"])
1333  for warpExpRef in warpRefList:
1334  if isinstance(warpExpRef, DeferredDatasetHandle):
1335  # Gen 3 API
1336  psf = warpExpRef.get(component="psf")
1337  else:
1338  # Gen 2 API. Delete this when Gen 2 retired
1339  psf = warpExpRef.get(tempExpName).getPsf()
1340  visit = warpExpRef.dataId["visit"]
1341  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1342  psfSizes[ccdVisits == visit] = psfSize
1343  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1344  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1345  # of the mean or median FWHM of the input exposures.
1346  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1347  goodPsfs = psfSizes <= sizeThreshold
1348  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1349  self.config.coaddPsf.makeControl())
1350  return psf
lsst::afw::image
lsst.pipe.tasks.dcrAssembleCoadd.selectCoaddPsf
def selectCoaddPsf(self, templateCoadd, warpRefList)
Definition: dcrAssembleCoadd.py:1309
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddConfig
Definition: assembleCoadd.py:1741
lsst.pipe.tasks.dcrAssembleCoadd.fillCoadd
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
Definition: dcrAssembleCoadd.py:1070
lsst::ip::diffim::dcrModel
lsst.pipe.tasks.dcrAssembleCoadd.dcrAssembleSubregion
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
Definition: dcrAssembleCoadd.py:805
lsst.pipe.tasks.dcrAssembleCoadd.calculateGain
def calculateGain(self, convergenceList, gainList)
Definition: dcrAssembleCoadd.py:1128
lsst.pipe.tasks.dcrAssembleCoadd.calculateSingleConvergence
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
Definition: dcrAssembleCoadd.py:1010
lsst.pipe.tasks.dcrAssembleCoadd.DcrAssembleCoaddConnections
Definition: dcrAssembleCoadd.py:49
lsst.pipe.tasks.coaddBase.makeSkyInfo
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:279
lsst.pipe.tasks.dcrAssembleCoadd.calculateModelWeights
def calculateModelWeights(self, dcrModels, dcrBBox)
Definition: dcrAssembleCoadd.py:1209
lsst.pipe.tasks.assembleCoadd.CompareWarpAssembleCoaddTask
Definition: assembleCoadd.py:1888
lsst.pipe.tasks.dcrAssembleCoadd.calculateConvergence
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
Definition: dcrAssembleCoadd.py:967
lsst.pipe.tasks.assembleCoadd.run
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, altMaskList=None, mask=None, supplementaryData=None)
Definition: assembleCoadd.py:721
lsst.pipe.tasks.dcrAssembleCoadd.dcrResiduals
def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth)
Definition: dcrAssembleCoadd.py:879
lsst::meas::base
lsst.pipe.tasks.dcrAssembleCoadd.applyModelWeights
def applyModelWeights(self, modelImages, refImage, modelWeights)
Definition: dcrAssembleCoadd.py:1242
lsst.pipe.tasks.dcrAssembleCoadd.newModelFromResidual
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
Definition: dcrAssembleCoadd.py:911
lsst::afw::table
lsst::utils
lsst::pex::config
lsst::geom
lsst::ip::diffim::dcrModel::DcrModel
lsst.pipe.tasks.assembleCoadd.makeSupplementaryDataGen3
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
Definition: assembleCoadd.py:565
lsst::geom::Box2I
lsst::skymap
lsst.pipe::base
lsst::meas::algorithms
lsst.pipe.tasks.dcrAssembleCoadd.stackCoadd
def stackCoadd(self, dcrCoadds)
Definition: dcrAssembleCoadd.py:1051
Extent< int, 2 >
lsst::coadd::utils
lsst.pipe.tasks.dcrAssembleCoadd.loadSubExposures
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
Definition: dcrAssembleCoadd.py:1262