lsst.pipe.tasks  21.0.0-165-g4927fbd8+b0a7cc4645
dcrAssembleCoadd.py
Go to the documentation of this file.
1 # This file is part of pipe_tasks.
2 #
3 # LSST Data Management System
4 # This product includes software developed by the
5 # LSST Project (http://www.lsst.org/).
6 # See COPYRIGHT file at the top of the source tree.
7 #
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
12 #
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU General Public License for more details.
17 #
18 # You should have received a copy of the LSST License Statement and
19 # the GNU General Public License along with this program. If not,
20 # see <https://www.lsstcorp.org/LegalNotices/>.
21 #
22 
23 from math import ceil
24 import numpy as np
25 from scipy import ndimage
26 import lsst.geom as geom
27 import lsst.afw.image as afwImage
28 import lsst.afw.table as afwTable
29 import lsst.coadd.utils as coaddUtils
30 from lsst.daf.butler import DeferredDatasetHandle
31 from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32 import lsst.meas.algorithms as measAlg
33 from lsst.meas.base import SingleFrameMeasurementTask
34 import lsst.pex.config as pexConfig
35 import lsst.pipe.base as pipeBase
36 import lsst.utils as utils
37 from lsst.skymap import BaseSkyMap
38 from lsst.utils.timer import timeMethod
39 from .assembleCoadd import (AssembleCoaddTask,
40  CompareWarpAssembleCoaddConfig,
41  CompareWarpAssembleCoaddTask)
42 from .coaddBase import makeSkyInfo
43 from .measurePsf import MeasurePsfTask
44 
45 __all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
46 
47 
48 class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
49  dimensions=("tract", "patch", "band", "skymap"),
50  defaultTemplates={"inputCoaddName": "deep",
51  "outputCoaddName": "dcr",
52  "warpType": "direct",
53  "warpTypeSuffix": "",
54  "fakesType": ""}):
55  inputWarps = pipeBase.connectionTypes.Input(
56  doc=("Input list of warps to be assembled i.e. stacked."
57  "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
58  name="{inputCoaddName}Coadd_{warpType}Warp",
59  storageClass="ExposureF",
60  dimensions=("tract", "patch", "skymap", "visit", "instrument"),
61  deferLoad=True,
62  multiple=True
63  )
64  skyMap = pipeBase.connectionTypes.Input(
65  doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
66  name=BaseSkyMap.SKYMAP_DATASET_TYPE_NAME,
67  storageClass="SkyMap",
68  dimensions=("skymap", ),
69  )
70  brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
71  doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
72  " BRIGHT_OBJECT."),
73  name="brightObjectMask",
74  storageClass="ObjectMaskCatalog",
75  dimensions=("tract", "patch", "skymap", "band"),
76  )
77  templateExposure = pipeBase.connectionTypes.Input(
78  doc="Input coadded exposure, produced by previous call to AssembleCoadd",
79  name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
80  storageClass="ExposureF",
81  dimensions=("tract", "patch", "skymap", "band"),
82  )
83  dcrCoadds = pipeBase.connectionTypes.Output(
84  doc="Output coadded exposure, produced by stacking input warps",
85  name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
86  storageClass="ExposureF",
87  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
88  multiple=True,
89  )
90  dcrNImages = pipeBase.connectionTypes.Output(
91  doc="Output image of number of input images per pixel",
92  name="{outputCoaddName}Coadd_nImage",
93  storageClass="ImageU",
94  dimensions=("tract", "patch", "skymap", "band", "subfilter"),
95  multiple=True,
96  )
97 
98  def __init__(self, *, config=None):
99  super().__init__(config=config)
100  if not config.doWrite:
101  self.outputs.remove("dcrCoadds")
102 
103 
104 class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
105  pipelineConnections=DcrAssembleCoaddConnections):
106  dcrNumSubfilters = pexConfig.Field(
107  dtype=int,
108  doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
109  default=3,
110  )
111  maxNumIter = pexConfig.Field(
112  dtype=int,
113  optional=True,
114  doc="Maximum number of iterations of forward modeling.",
115  default=None,
116  )
117  minNumIter = pexConfig.Field(
118  dtype=int,
119  optional=True,
120  doc="Minimum number of iterations of forward modeling.",
121  default=None,
122  )
123  convergenceThreshold = pexConfig.Field(
124  dtype=float,
125  doc="Target relative change in convergence between iterations of forward modeling.",
126  default=0.001,
127  )
128  useConvergence = pexConfig.Field(
129  dtype=bool,
130  doc="Use convergence test as a forward modeling end condition?"
131  "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
132  default=True,
133  )
134  baseGain = pexConfig.Field(
135  dtype=float,
136  optional=True,
137  doc="Relative weight to give the new solution vs. the last solution when updating the model."
138  "A value of 1.0 gives equal weight to both solutions."
139  "Small values imply slower convergence of the solution, but can "
140  "help prevent overshooting and failures in the fit."
141  "If ``baseGain`` is None, a conservative gain "
142  "will be calculated from the number of subfilters. ",
143  default=None,
144  )
145  useProgressiveGain = pexConfig.Field(
146  dtype=bool,
147  doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
148  "When calculating the next gain, we use up to 5 previous gains and convergence values."
149  "Can be set to False to force the model to change at the rate of ``baseGain``. ",
150  default=True,
151  )
152  doAirmassWeight = pexConfig.Field(
153  dtype=bool,
154  doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
155  default=False,
156  )
157  modelWeightsWidth = pexConfig.Field(
158  dtype=float,
159  doc="Width of the region around detected sources to include in the DcrModel.",
160  default=3,
161  )
162  useModelWeights = pexConfig.Field(
163  dtype=bool,
164  doc="Width of the region around detected sources to include in the DcrModel.",
165  default=True,
166  )
167  splitSubfilters = pexConfig.Field(
168  dtype=bool,
169  doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
170  "Instead of at the midpoint",
171  default=True,
172  )
173  splitThreshold = pexConfig.Field(
174  dtype=float,
175  doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
176  "Set to 0 to always split the subfilters.",
177  default=0.1,
178  )
179  regularizeModelIterations = pexConfig.Field(
180  dtype=float,
181  doc="Maximum relative change of the model allowed between iterations."
182  "Set to zero to disable.",
183  default=2.,
184  )
185  regularizeModelFrequency = pexConfig.Field(
186  dtype=float,
187  doc="Maximum relative change of the model allowed between subfilters."
188  "Set to zero to disable.",
189  default=4.,
190  )
191  convergenceMaskPlanes = pexConfig.ListField(
192  dtype=str,
193  default=["DETECTED"],
194  doc="Mask planes to use to calculate convergence."
195  )
196  regularizationWidth = pexConfig.Field(
197  dtype=int,
198  default=2,
199  doc="Minimum radius of a region to include in regularization, in pixels."
200  )
201  imageInterpOrder = pexConfig.Field(
202  dtype=int,
203  doc="The order of the spline interpolation used to shift the image plane.",
204  default=3,
205  )
206  accelerateModel = pexConfig.Field(
207  dtype=float,
208  doc="Factor to amplify the differences between model planes by to speed convergence.",
209  default=3,
210  )
211  doCalculatePsf = pexConfig.Field(
212  dtype=bool,
213  doc="Set to detect stars and recalculate the PSF from the final coadd."
214  "Otherwise the PSF is estimated from a selection of the best input exposures",
215  default=False,
216  )
217  detectPsfSources = pexConfig.ConfigurableField(
218  target=measAlg.SourceDetectionTask,
219  doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
220  )
221  measurePsfSources = pexConfig.ConfigurableField(
222  target=SingleFrameMeasurementTask,
223  doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
224  )
225  measurePsf = pexConfig.ConfigurableField(
226  target=MeasurePsfTask,
227  doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
228  )
229  effectiveWavelength = pexConfig.Field(
230  doc="Effective wavelength of the filter, in nm."
231  "Required if transmission curves aren't used."
232  "Support for using transmission curves is to be added in DM-13668.",
233  dtype=float,
234  )
235  bandwidth = pexConfig.Field(
236  doc="Bandwidth of the physical filter, in nm."
237  "Required if transmission curves aren't used."
238  "Support for using transmission curves is to be added in DM-13668.",
239  dtype=float,
240  )
241 
242  def setDefaults(self):
243  CompareWarpAssembleCoaddConfig.setDefaults(self)
244  self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
245  self.doNImage = True
246  self.assembleStaticSkyModel.warpType = self.warpType
247  # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
248  self.assembleStaticSkyModel.doNImage = False
249  self.assembleStaticSkyModel.doWrite = False
250  self.detectPsfSources.returnOriginalFootprints = False
251  self.detectPsfSources.thresholdPolarity = "positive"
252  # Only use bright sources for PSF measurement
253  self.detectPsfSources.thresholdValue = 50
254  self.detectPsfSources.nSigmaToGrow = 2
255  # A valid star for PSF measurement should at least fill 5x5 pixels
256  self.detectPsfSources.minPixels = 25
257  # Use the variance plane to calculate signal to noise
258  self.detectPsfSources.thresholdType = "pixel_stdev"
259  # The signal to noise limit is good enough, while the flux limit is set
260  # in dimensionless units and may not be appropriate for all data sets.
261  self.measurePsf.starSelector["objectSize"].doFluxLimit = False
262 
263 
264 class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
265  """Assemble DCR coadded images from a set of warps.
266 
267  Attributes
268  ----------
269  bufferSize : `int`
270  The number of pixels to grow each subregion by to allow for DCR.
271 
272  Notes
273  -----
274  As with AssembleCoaddTask, we want to assemble a coadded image from a set of
275  Warps (also called coadded temporary exposures), including the effects of
276  Differential Chromatic Refraction (DCR).
277  For full details of the mathematics and algorithm, please see
278  DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
279 
280  This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
281  each subfilter used in the iterative calculation.
282  It begins by dividing the bandpass-defining filter into N equal bandwidth
283  "subfilters", and divides the flux in each pixel from an initial coadd
284  equally into each as a "dcrModel". Because the airmass and parallactic
285  angle of each individual exposure is known, we can calculate the shift
286  relative to the center of the band in each subfilter due to DCR. For each
287  exposure we apply this shift as a linear transformation to the dcrModels
288  and stack the results to produce a DCR-matched exposure. The matched
289  exposures are subtracted from the input exposures to produce a set of
290  residual images, and these residuals are reverse shifted for each
291  exposures' subfilters and stacked. The shifted and stacked residuals are
292  added to the dcrModels to produce a new estimate of the flux in each pixel
293  within each subfilter. The dcrModels are solved for iteratively, which
294  continues until the solution from a new iteration improves by less than
295  a set percentage, or a maximum number of iterations is reached.
296  Two forms of regularization are employed to reduce unphysical results.
297  First, the new solution is averaged with the solution from the previous
298  iteration, which mitigates oscillating solutions where the model
299  overshoots with alternating very high and low values.
300  Second, a common degeneracy when the data have a limited range of airmass or
301  parallactic angle values is for one subfilter to be fit with very low or
302  negative values, while another subfilter is fit with very high values. This
303  typically appears in the form of holes next to sources in one subfilter,
304  and corresponding extended wings in another. Because each subfilter has
305  a narrow bandwidth we assume that physical sources that are above the noise
306  level will not vary in flux by more than a factor of `frequencyClampFactor`
307  between subfilters, and pixels that have flux deviations larger than that
308  factor will have the excess flux distributed evenly among all subfilters.
309  If `splitSubfilters` is set, then each subfilter will be further sub-
310  divided during the forward modeling step (only). This approximates using
311  a higher number of subfilters that may be necessary for high airmass
312  observations, but does not increase the number of free parameters in the
313  fit. This is needed when there are high airmass observations which would
314  otherwise have significant DCR even within a subfilter. Because calculating
315  the shifted images takes most of the time, splitting the subfilters is
316  turned off by way of the `splitThreshold` option for low-airmass
317  observations that do not suffer from DCR within a subfilter.
318  """
319 
320  ConfigClass = DcrAssembleCoaddConfig
321  _DefaultName = "dcrAssembleCoadd"
322 
323  def __init__(self, *args, **kwargs):
324  super().__init__(*args, **kwargs)
325  if self.config.doCalculatePsf:
326  self.schema = afwTable.SourceTable.makeMinimalSchema()
327  self.makeSubtask("detectPsfSources", schema=self.schema)
328  self.makeSubtask("measurePsfSources", schema=self.schema)
329  self.makeSubtask("measurePsf", schema=self.schema)
330 
331  @utils.inheritDoc(pipeBase.PipelineTask)
332  def runQuantum(self, butlerQC, inputRefs, outputRefs):
333  # Docstring to be formatted with info from PipelineTask.runQuantum
334  """
335  Notes
336  -----
337  Assemble a coadd from a set of Warps.
338 
339  PipelineTask (Gen3) entry point to Coadd a set of Warps.
340  Analogous to `runDataRef`, it prepares all the data products to be
341  passed to `run`, and processes the results before returning a struct
342  of results to be written out. AssembleCoadd cannot fit all Warps in memory.
343  Therefore, its inputs are accessed subregion by subregion
344  by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
345  `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
346  correspond to an update in `runDataRef` while both entry points
347  are used.
348  """
349  inputData = butlerQC.get(inputRefs)
350 
351  # Construct skyInfo expected by run
352  # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
353  skyMap = inputData["skyMap"]
354  outputDataId = butlerQC.quantum.dataId
355 
356  inputData['skyInfo'] = makeSkyInfo(skyMap,
357  tractId=outputDataId['tract'],
358  patchId=outputDataId['patch'])
359 
360  # Construct list of input Deferred Datasets
361  # These quack a bit like like Gen2 DataRefs
362  warpRefList = inputData['inputWarps']
363  # Perform same middle steps as `runDataRef` does
364  inputs = self.prepareInputs(warpRefList)
365  self.log.info("Found %d %s", len(inputs.tempExpRefList),
366  self.getTempExpDatasetName(self.warpType))
367  if len(inputs.tempExpRefList) == 0:
368  self.log.warning("No coadd temporary exposures found")
369  return
370 
371  supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
372  retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
373  inputs.weightList, supplementaryData=supplementaryData)
374 
375  inputData.setdefault('brightObjectMask', None)
376  for subfilter in range(self.config.dcrNumSubfilters):
377  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
378  retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
379  self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
380 
381  if self.config.doWrite:
382  butlerQC.put(retStruct, outputRefs)
383  return retStruct
384 
385  @timeMethod
386  def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
387  """Assemble a coadd from a set of warps.
388 
389  Coadd a set of Warps. Compute weights to be applied to each Warp and
390  find scalings to match the photometric zeropoint to a reference Warp.
391  Assemble the Warps using run method.
392  Forward model chromatic effects across multiple subfilters,
393  and subtract from the input Warps to build sets of residuals.
394  Use the residuals to construct a new ``DcrModel`` for each subfilter,
395  and iterate until the model converges.
396  Interpolate over NaNs and optionally write the coadd to disk.
397  Return the coadded exposure.
398 
399  Parameters
400  ----------
401  dataRef : `lsst.daf.persistence.ButlerDataRef`
402  Data reference defining the patch for coaddition and the
403  reference Warp
404  selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
405  List of data references to warps. Data to be coadded will be
406  selected from this list based on overlap with the patch defined by
407  the data reference.
408 
409  Returns
410  -------
411  results : `lsst.pipe.base.Struct`
412  The Struct contains the following fields:
413 
414  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
415  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
416  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
417  - ``dcrNImages``: `list` of exposure count images for each subfilter
418  """
419  if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
420  raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
421 
422  skyInfo = self.getSkyInfo(dataRef)
423  if warpRefList is None:
424  calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
425  if len(calExpRefList) == 0:
426  self.log.warning("No exposures to coadd")
427  return
428  self.log.info("Coadding %d exposures", len(calExpRefList))
429 
430  warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
431 
432  inputData = self.prepareInputs(warpRefList)
433  self.log.info("Found %d %s", len(inputData.tempExpRefList),
434  self.getTempExpDatasetName(self.warpType))
435  if len(inputData.tempExpRefList) == 0:
436  self.log.warning("No coadd temporary exposures found")
437  return
438 
439  supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
440 
441  results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
442  inputData.weightList, supplementaryData=supplementaryData)
443  if results is None:
444  self.log.warning("Could not construct DcrModel for patch %s: no data to coadd.",
445  skyInfo.patchInfo.getIndex())
446  return
447 
448  if self.config.doCalculatePsf:
449  self.measureCoaddPsf(results.coaddExposure)
450  brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
451  for subfilter in range(self.config.dcrNumSubfilters):
452  # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
453  results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
454  self.processResults(results.dcrCoadds[subfilter],
455  brightObjectMasks=brightObjects, dataId=dataRef.dataId)
456  if self.config.doWrite:
457  self.log.info("Persisting dcrCoadd")
458  dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
459  numSubfilters=self.config.dcrNumSubfilters)
460  if self.config.doNImage and results.dcrNImages is not None:
461  dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
462  numSubfilters=self.config.dcrNumSubfilters)
463 
464  return results
465 
466  @utils.inheritDoc(AssembleCoaddTask)
467  def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
468  """Load the previously-generated template coadd.
469 
470  This can be removed entirely once we no longer support the Gen 2 butler.
471 
472  Returns
473  -------
474  templateCoadd : `lsst.pipe.base.Struct`
475  Result struct with components:
476 
477  - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
478  """
479  templateCoadd = butlerQC.get(inputRefs.templateExposure)
480 
481  return pipeBase.Struct(templateCoadd=templateCoadd)
482 
483  def measureCoaddPsf(self, coaddExposure):
484  """Detect sources on the coadd exposure and measure the final PSF.
485 
486  Parameters
487  ----------
488  coaddExposure : `lsst.afw.image.Exposure`
489  The final coadded exposure.
490  """
491  table = afwTable.SourceTable.make(self.schema)
492  detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
493  coaddSources = detResults.sources
494  self.measurePsfSources.run(
495  measCat=coaddSources,
496  exposure=coaddExposure
497  )
498  # Measure the PSF on the stacked subfilter coadds if possible.
499  # We should already have a decent estimate of the coadd PSF, however,
500  # so in case of any errors simply log them as a warning and use the
501  # default PSF.
502  try:
503  psfResults = self.measurePsf.run(coaddExposure, coaddSources)
504  except Exception as e:
505  self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e)
506  else:
507  coaddExposure.setPsf(psfResults.psf)
508 
509  def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
510  """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
511 
512  Sets the property ``bufferSize``.
513 
514  Parameters
515  ----------
516  templateCoadd : `lsst.afw.image.ExposureF`
517  The initial coadd exposure before accounting for DCR.
518  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
519  `lsst.daf.persistence.ButlerDataRef`
520  The data references to the input warped exposures.
521  weightList : `list` of `float`
522  The weight to give each input exposure in the coadd
523  Will be modified in place if ``doAirmassWeight`` is set.
524 
525  Returns
526  -------
527  dcrModels : `lsst.pipe.tasks.DcrModel`
528  Best fit model of the true sky after correcting chromatic effects.
529 
530  Raises
531  ------
532  NotImplementedError
533  If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
534  """
535  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
536  filterLabel = templateCoadd.getFilterLabel()
537  tempExpName = self.getTempExpDatasetName(self.warpType)
538  dcrShifts = []
539  airmassDict = {}
540  angleDict = {}
541  psfSizeDict = {}
542  for visitNum, warpExpRef in enumerate(warpRefList):
543  if isinstance(warpExpRef, DeferredDatasetHandle):
544  # Gen 3 API
545  visitInfo = warpExpRef.get(component="visitInfo")
546  psf = warpExpRef.get(component="psf")
547  else:
548  # Gen 2 API. Delete this when Gen 2 retired
549  visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
550  psf = warpExpRef.get(tempExpName).getPsf()
551  visit = warpExpRef.dataId["visit"]
552  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
553  airmass = visitInfo.getBoresightAirmass()
554  parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
555  airmassDict[visit] = airmass
556  angleDict[visit] = parallacticAngle
557  psfSizeDict[visit] = psfSize
558  if self.config.doAirmassWeight:
559  weightList[visitNum] *= airmass
560  dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
561  self.config.effectiveWavelength,
562  self.config.bandwidth,
563  self.config.dcrNumSubfilters))))
564  self.log.info("Selected airmasses:\n%s", airmassDict)
565  self.log.info("Selected parallactic angles:\n%s", angleDict)
566  self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
567  self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
568  try:
569  psf = self.selectCoaddPsf(templateCoadd, warpRefList)
570  except Exception as e:
571  self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e)
572  else:
573  psf = templateCoadd.getPsf()
574  dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
575  self.config.dcrNumSubfilters,
576  effectiveWavelength=self.config.effectiveWavelength,
577  bandwidth=self.config.bandwidth,
578  filterLabel=filterLabel,
579  psf=psf)
580  return dcrModels
581 
582  @timeMethod
583  def run(self, skyInfo, warpRefList, imageScalerList, weightList,
584  supplementaryData=None):
585  """Assemble the coadd.
586 
587  Requires additional inputs Struct ``supplementaryData`` to contain a
588  ``templateCoadd`` that serves as the model of the static sky.
589 
590  Find artifacts and apply them to the warps' masks creating a list of
591  alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
592  Then pass these alternative masks to the base class's assemble method.
593 
594  Divide the ``templateCoadd`` evenly between each subfilter of a
595  ``DcrModel`` as the starting best estimate of the true wavelength-
596  dependent sky. Forward model the ``DcrModel`` using the known
597  chromatic effects in each subfilter and calculate a convergence metric
598  based on how well the modeled template matches the input warps. If
599  the convergence has not yet reached the desired threshold, then shift
600  and stack the residual images to build a new ``DcrModel``. Apply
601  conditioning to prevent oscillating solutions between iterations or
602  between subfilters.
603 
604  Once the ``DcrModel`` reaches convergence or the maximum number of
605  iterations has been reached, fill the metadata for each subfilter
606  image and make them proper ``coaddExposure``s.
607 
608  Parameters
609  ----------
610  skyInfo : `lsst.pipe.base.Struct`
611  Patch geometry information, from getSkyInfo
612  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
613  `lsst.daf.persistence.ButlerDataRef`
614  The data references to the input warped exposures.
615  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
616  The image scalars correct for the zero point of the exposures.
617  weightList : `list` of `float`
618  The weight to give each input exposure in the coadd
619  supplementaryData : `lsst.pipe.base.Struct`
620  Result struct returned by ``makeSupplementaryData`` with components:
621 
622  - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
623 
624  Returns
625  -------
626  result : `lsst.pipe.base.Struct`
627  Result struct with components:
628 
629  - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
630  - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
631  - ``dcrCoadds``: `list` of coadded exposures for each subfilter
632  - ``dcrNImages``: `list` of exposure count images for each subfilter
633  """
634  minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
635  maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
636  templateCoadd = supplementaryData.templateCoadd
637  baseMask = templateCoadd.mask.clone()
638  # The variance plane is for each subfilter
639  # and should be proportionately lower than the full-band image
640  baseVariance = templateCoadd.variance.clone()
641  baseVariance /= self.config.dcrNumSubfilters
642  spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
643  # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
644  templateCoadd.setMask(baseMask)
645  badMaskPlanes = self.config.badMaskPlanes[:]
646  # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
647  # This is because pixels in observations that are significantly affect by DCR
648  # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
649  # but those are necessary to constrain the DCR model.
650  badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
651 
652  stats = self.prepareStats(mask=badPixelMask)
653  dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
654  if self.config.doNImage:
655  dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
656  spanSetMaskList, stats.ctrl)
657  nImage = afwImage.ImageU(skyInfo.bbox)
658  # Note that this nImage will be a factor of dcrNumSubfilters higher than
659  # the nImage returned by assembleCoadd for most pixels. This is because each
660  # subfilter may have a different nImage, and fractional values are not allowed.
661  for dcrNImage in dcrNImages:
662  nImage += dcrNImage
663  else:
664  dcrNImages = None
665 
666  subregionSize = geom.Extent2I(*self.config.subregionSize)
667  nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
668  * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
669  subIter = 0
670  for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
671  modelIter = 0
672  subIter += 1
673  self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
674  skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
675  dcrBBox = geom.Box2I(subBBox)
676  dcrBBox.grow(self.bufferSize)
677  dcrBBox.clip(dcrModels.bbox)
678  modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
679  subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
680  imageScalerList, spanSetMaskList)
681  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
682  warpRefList, weightList, stats.ctrl)
683  self.log.info("Initial convergence : %s", convergenceMetric)
684  convergenceList = [convergenceMetric]
685  gainList = []
686  convergenceCheck = 1.
687  refImage = templateCoadd.image
688  while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
689  gain = self.calculateGain(convergenceList, gainList)
690  self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
691  stats.ctrl, convergenceMetric, gain,
692  modelWeights, refImage, dcrWeights)
693  if self.config.useConvergence:
694  convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
695  warpRefList, weightList, stats.ctrl)
696  if convergenceMetric == 0:
697  self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
698  "most likely due to there being no valid data in the region.",
699  skyInfo.patchInfo.getIndex(), subIter)
700  break
701  convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
702  if (convergenceCheck < 0) & (modelIter > minNumIter):
703  self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum "
704  "iterations or desired convergence improvement of %s."
705  " Divergence: %s",
706  skyInfo.patchInfo.getIndex(), subIter,
707  self.config.convergenceThreshold, convergenceCheck)
708  break
709  convergenceList.append(convergenceMetric)
710  if modelIter > maxNumIter:
711  if self.config.useConvergence:
712  self.log.warning("Coadd patch %s subregion %s reached maximum iterations "
713  "before reaching desired convergence improvement of %s."
714  " Final convergence improvement: %s",
715  skyInfo.patchInfo.getIndex(), subIter,
716  self.config.convergenceThreshold, convergenceCheck)
717  break
718 
719  if self.config.useConvergence:
720  self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
721  modelIter, convergenceMetric, 100.*convergenceCheck, gain)
722  modelIter += 1
723  else:
724  if self.config.useConvergence:
725  self.log.info("Coadd patch %s subregion %s finished with "
726  "convergence metric %s after %s iterations",
727  skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
728  else:
729  self.log.info("Coadd patch %s subregion %s finished after %s iterations",
730  skyInfo.patchInfo.getIndex(), subIter, modelIter)
731  if self.config.useConvergence and convergenceMetric > 0:
732  self.log.info("Final convergence improvement was %.4f%% overall",
733  100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
734 
735  dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
736  calibration=self.scaleZeroPoint.getPhotoCalib(),
737  coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
738  mask=baseMask,
739  variance=baseVariance)
740  coaddExposure = self.stackCoadd(dcrCoadds)
741  return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
742  dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
743 
744  def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
745  """Calculate the number of exposures contributing to each subfilter.
746 
747  Parameters
748  ----------
749  dcrModels : `lsst.pipe.tasks.DcrModel`
750  Best fit model of the true sky after correcting chromatic effects.
751  bbox : `lsst.geom.box.Box2I`
752  Bounding box of the patch to coadd.
753  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
754  `lsst.daf.persistence.ButlerDataRef`
755  The data references to the input warped exposures.
756  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
757  Each element of the `dict` contains the new mask plane name
758  (e.g. "CLIPPED and/or "NO_DATA") as the key,
759  and the list of SpanSets to apply to the mask.
760  statsCtrl : `lsst.afw.math.StatisticsControl`
761  Statistics control object for coadd
762 
763  Returns
764  -------
765  dcrNImages : `list` of `lsst.afw.image.ImageU`
766  List of exposure count images for each subfilter
767  dcrWeights : `list` of `lsst.afw.image.ImageF`
768  Per-pixel weights for each subfilter.
769  Equal to 1/(number of unmasked images contributing to each pixel).
770  """
771  dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
772  dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
773  tempExpName = self.getTempExpDatasetName(self.warpType)
774  for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
775  if isinstance(warpExpRef, DeferredDatasetHandle):
776  # Gen 3 API
777  exposure = warpExpRef.get(parameters={'bbox': bbox})
778  else:
779  # Gen 2 API. Delete this when Gen 2 retired
780  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
781  visitInfo = exposure.getInfo().getVisitInfo()
782  wcs = exposure.getInfo().getWcs()
783  mask = exposure.mask
784  if altMaskSpans is not None:
785  self.applyAltMaskPlanes(mask, altMaskSpans)
786  weightImage = np.zeros_like(exposure.image.array)
787  weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
788  # The weights must be shifted in exactly the same way as the residuals,
789  # because they will be used as the denominator in the weighted average of residuals.
790  weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
791  dcrModels.effectiveWavelength, dcrModels.bandwidth)
792  for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
793  dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
794  dcrWeight.array += shiftedWeights
795  # Exclude any pixels that don't have at least one exposure contributing in all subfilters
796  weightsThreshold = 1.
797  goodPix = dcrWeights[0].array > weightsThreshold
798  for weights in dcrWeights[1:]:
799  goodPix = (weights.array > weightsThreshold) & goodPix
800  for subfilter in range(self.config.dcrNumSubfilters):
801  dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
802  dcrWeights[subfilter].array[~goodPix] = 0.
803  dcrNImages[subfilter].array[~goodPix] = 0
804  return (dcrNImages, dcrWeights)
805 
806  def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
807  statsCtrl, convergenceMetric,
808  gain, modelWeights, refImage, dcrWeights):
809  """Assemble the DCR coadd for a sub-region.
810 
811  Build a DCR-matched template for each input exposure, then shift the
812  residuals according to the DCR in each subfilter.
813  Stack the shifted residuals and apply them as a correction to the
814  solution from the previous iteration.
815  Restrict the new model solutions from varying by more than a factor of
816  `modelClampFactor` from the last solution, and additionally restrict the
817  individual subfilter models from varying by more than a factor of
818  `frequencyClampFactor` from their average.
819  Finally, mitigate potentially oscillating solutions by averaging the new
820  solution with the solution from the previous iteration, weighted by
821  their convergence metric.
822 
823  Parameters
824  ----------
825  dcrModels : `lsst.pipe.tasks.DcrModel`
826  Best fit model of the true sky after correcting chromatic effects.
827  subExposures : `dict` of `lsst.afw.image.ExposureF`
828  The pre-loaded exposures for the current subregion.
829  bbox : `lsst.geom.box.Box2I`
830  Bounding box of the subregion to coadd.
831  dcrBBox : `lsst.geom.box.Box2I`
832  Sub-region of the coadd which includes a buffer to allow for DCR.
833  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
834  `lsst.daf.persistence.ButlerDataRef`
835  The data references to the input warped exposures.
836  statsCtrl : `lsst.afw.math.StatisticsControl`
837  Statistics control object for coadd
838  convergenceMetric : `float`
839  Quality of fit metric for the matched templates of the input images.
840  gain : `float`, optional
841  Relative weight to give the new solution when updating the model.
842  modelWeights : `numpy.ndarray` or `float`
843  A 2D array of weight values that tapers smoothly to zero away from detected sources.
844  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
845  refImage : `lsst.afw.image.Image`
846  A reference image used to supply the default pixel values.
847  dcrWeights : `list` of `lsst.afw.image.Image`
848  Per-pixel weights for each subfilter.
849  Equal to 1/(number of unmasked images contributing to each pixel).
850  """
851  residualGeneratorList = []
852 
853  for warpExpRef in warpRefList:
854  visit = warpExpRef.dataId["visit"]
855  exposure = subExposures[visit]
856  visitInfo = exposure.getInfo().getVisitInfo()
857  wcs = exposure.getInfo().getWcs()
858  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
859  order=self.config.imageInterpOrder,
860  splitSubfilters=self.config.splitSubfilters,
861  splitThreshold=self.config.splitThreshold,
862  amplifyModel=self.config.accelerateModel)
863  residual = exposure.image.array - templateImage.array
864  # Note that the variance plane here is used to store weights, not the actual variance
865  residual *= exposure.variance.array
866  # The residuals are stored as a list of generators.
867  # This allows the residual for a given subfilter and exposure to be created
868  # on the fly, instead of needing to store them all in memory.
869  residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
870  dcrModels.effectiveWavelength,
871  dcrModels.bandwidth))
872 
873  dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
874  gain=gain,
875  modelWeights=modelWeights,
876  refImage=refImage,
877  dcrWeights=dcrWeights)
878  dcrModels.assign(dcrSubModelOut, bbox)
879 
880  def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
881  """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
882 
883  Parameters
884  ----------
885  residual : `numpy.ndarray`
886  The residual masked image for one exposure,
887  after subtracting the matched template
888  visitInfo : `lsst.afw.image.VisitInfo`
889  Metadata for the exposure.
890  wcs : `lsst.afw.geom.SkyWcs`
891  Coordinate system definition (wcs) for the exposure.
892 
893  Yields
894  ------
895  residualImage : `numpy.ndarray`
896  The residual image for the next subfilter, shifted for DCR.
897  """
898  # Pre-calculate the spline-filtered residual image, so that step can be
899  # skipped in the shift calculation in `applyDcr`.
900  filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
901  # Note that `splitSubfilters` is always turned off in the reverse direction.
902  # This option introduces additional blurring if applied to the residuals.
903  dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
904  splitSubfilters=False)
905  for dcr in dcrShift:
906  yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
907  doPrefilter=False, order=self.config.imageInterpOrder)
908 
909  def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
910  gain, modelWeights, refImage, dcrWeights):
911  """Calculate a new DcrModel from a set of image residuals.
912 
913  Parameters
914  ----------
915  dcrModels : `lsst.pipe.tasks.DcrModel`
916  Current model of the true sky after correcting chromatic effects.
917  residualGeneratorList : `generator` of `numpy.ndarray`
918  The residual image for the next subfilter, shifted for DCR.
919  dcrBBox : `lsst.geom.box.Box2I`
920  Sub-region of the coadd which includes a buffer to allow for DCR.
921  statsCtrl : `lsst.afw.math.StatisticsControl`
922  Statistics control object for coadd
923  gain : `float`
924  Relative weight to give the new solution when updating the model.
925  modelWeights : `numpy.ndarray` or `float`
926  A 2D array of weight values that tapers smoothly to zero away from detected sources.
927  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
928  refImage : `lsst.afw.image.Image`
929  A reference image used to supply the default pixel values.
930  dcrWeights : `list` of `lsst.afw.image.Image`
931  Per-pixel weights for each subfilter.
932  Equal to 1/(number of unmasked images contributing to each pixel).
933 
934  Returns
935  -------
936  dcrModel : `lsst.pipe.tasks.DcrModel`
937  New model of the true sky after correcting chromatic effects.
938  """
939  newModelImages = []
940  for subfilter, model in enumerate(dcrModels):
941  residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
942  residual = np.sum(residualsList, axis=0)
943  residual *= dcrWeights[subfilter][dcrBBox].array
944  # `MaskedImage`s only support in-place addition, so rename for readability
945  newModel = model[dcrBBox].clone()
946  newModel.array += residual
947  # Catch any invalid values
948  badPixels = ~np.isfinite(newModel.array)
949  newModel.array[badPixels] = model[dcrBBox].array[badPixels]
950  if self.config.regularizeModelIterations > 0:
951  dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
952  self.config.regularizeModelIterations,
953  self.config.regularizationWidth)
954  newModelImages.append(newModel)
955  if self.config.regularizeModelFrequency > 0:
956  dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
957  self.config.regularizeModelFrequency,
958  self.config.regularizationWidth)
959  dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
960  self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
961  return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
962  dcrModels.bandwidth, dcrModels.psf,
963  dcrModels.mask, dcrModels.variance)
964 
965  def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
966  """Calculate a quality of fit metric for the matched templates.
967 
968  Parameters
969  ----------
970  dcrModels : `lsst.pipe.tasks.DcrModel`
971  Best fit model of the true sky after correcting chromatic effects.
972  subExposures : `dict` of `lsst.afw.image.ExposureF`
973  The pre-loaded exposures for the current subregion.
974  bbox : `lsst.geom.box.Box2I`
975  Sub-region to coadd
976  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
977  `lsst.daf.persistence.ButlerDataRef`
978  The data references to the input warped exposures.
979  weightList : `list` of `float`
980  The weight to give each input exposure in the coadd
981  statsCtrl : `lsst.afw.math.StatisticsControl`
982  Statistics control object for coadd
983 
984  Returns
985  -------
986  convergenceMetric : `float`
987  Quality of fit metric for all input exposures, within the sub-region
988  """
989  significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
990  nSigma = 3.
991  significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
992  bufferSize=self.bufferSize)
993  if np.max(significanceImage) == 0:
994  significanceImage += 1.
995  weight = 0
996  metric = 0.
997  metricList = {}
998  for warpExpRef, expWeight in zip(warpRefList, weightList):
999  visit = warpExpRef.dataId["visit"]
1000  exposure = subExposures[visit][bbox]
1001  singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
1002  metric += singleMetric
1003  metricList[visit] = singleMetric
1004  weight += 1.
1005  self.log.info("Individual metrics:\n%s", metricList)
1006  return 1.0 if weight == 0.0 else metric/weight
1007 
1008  def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
1009  """Calculate a quality of fit metric for a single matched template.
1010 
1011  Parameters
1012  ----------
1013  dcrModels : `lsst.pipe.tasks.DcrModel`
1014  Best fit model of the true sky after correcting chromatic effects.
1015  exposure : `lsst.afw.image.ExposureF`
1016  The input warped exposure to evaluate.
1017  significanceImage : `numpy.ndarray`
1018  Array of weights for each pixel corresponding to its significance
1019  for the convergence calculation.
1020  statsCtrl : `lsst.afw.math.StatisticsControl`
1021  Statistics control object for coadd
1022 
1023  Returns
1024  -------
1025  convergenceMetric : `float`
1026  Quality of fit metric for one exposure, within the sub-region.
1027  """
1028  convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1029  templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1030  order=self.config.imageInterpOrder,
1031  splitSubfilters=self.config.splitSubfilters,
1032  splitThreshold=self.config.splitThreshold,
1033  amplifyModel=self.config.accelerateModel)
1034  diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1035  refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1036 
1037  finitePixels = np.isfinite(diffVals)
1038  goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1039  convergeMaskPixels = exposure.mask.array & convergeMask > 0
1040  usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1041  if np.sum(usePixels) == 0:
1042  metric = 0.
1043  else:
1044  diffUse = diffVals[usePixels]
1045  refUse = refVals[usePixels]
1046  metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1047  return metric
1048 
1049  def stackCoadd(self, dcrCoadds):
1050  """Add a list of sub-band coadds together.
1051 
1052  Parameters
1053  ----------
1054  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1055  A list of coadd exposures, each exposure containing
1056  the model for one subfilter.
1057 
1058  Returns
1059  -------
1060  coaddExposure : `lsst.afw.image.ExposureF`
1061  A single coadd exposure that is the sum of the sub-bands.
1062  """
1063  coaddExposure = dcrCoadds[0].clone()
1064  for coadd in dcrCoadds[1:]:
1065  coaddExposure.maskedImage += coadd.maskedImage
1066  return coaddExposure
1067 
1068  def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1069  mask=None, variance=None):
1070  """Create a list of coadd exposures from a list of masked images.
1071 
1072  Parameters
1073  ----------
1074  dcrModels : `lsst.pipe.tasks.DcrModel`
1075  Best fit model of the true sky after correcting chromatic effects.
1076  skyInfo : `lsst.pipe.base.Struct`
1077  Patch geometry information, from getSkyInfo
1078  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1079  `lsst.daf.persistence.ButlerDataRef`
1080  The data references to the input warped exposures.
1081  weightList : `list` of `float`
1082  The weight to give each input exposure in the coadd
1083  calibration : `lsst.afw.Image.PhotoCalib`, optional
1084  Scale factor to set the photometric calibration of an exposure.
1085  coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1086  A record of the observations that are included in the coadd.
1087  mask : `lsst.afw.image.Mask`, optional
1088  Optional mask to override the values in the final coadd.
1089  variance : `lsst.afw.image.Image`, optional
1090  Optional variance plane to override the values in the final coadd.
1091 
1092  Returns
1093  -------
1094  dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1095  A list of coadd exposures, each exposure containing
1096  the model for one subfilter.
1097  """
1098  dcrCoadds = []
1099  refModel = dcrModels.getReferenceImage()
1100  for model in dcrModels:
1101  if self.config.accelerateModel > 1:
1102  model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1103  coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1104  if calibration is not None:
1105  coaddExposure.setPhotoCalib(calibration)
1106  if coaddInputs is not None:
1107  coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1108  # Set the metadata for the coadd, including PSF and aperture corrections.
1109  self.assembleMetadata(coaddExposure, warpRefList, weightList)
1110  # Overwrite the PSF
1111  coaddExposure.setPsf(dcrModels.psf)
1112  coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1113  maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1114  maskedImage.image = model
1115  maskedImage.mask = dcrModels.mask
1116  maskedImage.variance = dcrModels.variance
1117  coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1118  coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1119  if mask is not None:
1120  coaddExposure.setMask(mask)
1121  if variance is not None:
1122  coaddExposure.setVariance(variance)
1123  dcrCoadds.append(coaddExposure)
1124  return dcrCoadds
1125 
1126  def calculateGain(self, convergenceList, gainList):
1127  """Calculate the gain to use for the current iteration.
1128 
1129  After calculating a new DcrModel, each value is averaged with the
1130  value in the corresponding pixel from the previous iteration. This
1131  reduces oscillating solutions that iterative techniques are plagued by,
1132  and speeds convergence. By far the biggest changes to the model
1133  happen in the first couple iterations, so we can also use a more
1134  aggressive gain later when the model is changing slowly.
1135 
1136  Parameters
1137  ----------
1138  convergenceList : `list` of `float`
1139  The quality of fit metric from each previous iteration.
1140  gainList : `list` of `float`
1141  The gains used in each previous iteration: appended with the new
1142  gain value.
1143  Gains are numbers between ``self.config.baseGain`` and 1.
1144 
1145  Returns
1146  -------
1147  gain : `float`
1148  Relative weight to give the new solution when updating the model.
1149  A value of 1.0 gives equal weight to both solutions.
1150 
1151  Raises
1152  ------
1153  ValueError
1154  If ``len(convergenceList) != len(gainList)+1``.
1155  """
1156  nIter = len(convergenceList)
1157  if nIter != len(gainList) + 1:
1158  raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1159  % (len(convergenceList), len(gainList)))
1160 
1161  if self.config.baseGain is None:
1162  # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1163  # The more subfilters being modeled, the lower the gain should be.
1164  baseGain = 1./(self.config.dcrNumSubfilters - 1)
1165  else:
1166  baseGain = self.config.baseGain
1167 
1168  if self.config.useProgressiveGain and nIter > 2:
1169  # To calculate the best gain to use, compare the past gains that have been used
1170  # with the resulting convergences to estimate the best gain to use.
1171  # Algorithmically, this is a Kalman filter.
1172  # If forward modeling proceeds perfectly, the convergence metric should
1173  # asymptotically approach a final value.
1174  # We can estimate that value from the measured changes in convergence
1175  # weighted by the gains used in each previous iteration.
1176  estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1177  for i in range(nIter - 1)]
1178  # The convergence metric is strictly positive, so if the estimated final convergence is
1179  # less than zero, force it to zero.
1180  estFinalConv = np.array(estFinalConv)
1181  estFinalConv[estFinalConv < 0] = 0
1182  # Because the estimate may slowly change over time, only use the most recent measurements.
1183  estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1184  lastGain = gainList[-1]
1185  lastConv = convergenceList[-2]
1186  newConv = convergenceList[-1]
1187  # The predicted convergence is the value we would get if the new model calculated
1188  # in the previous iteration was perfect. Recall that the updated model that is
1189  # actually used is the gain-weighted average of the new and old model,
1190  # so the convergence would be similarly weighted.
1191  predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1192  # If the measured and predicted convergence are very close, that indicates
1193  # that our forward model is accurate and we can use a more aggressive gain
1194  # If the measured convergence is significantly worse (or better!) than predicted,
1195  # that indicates that the model is not converging as expected and
1196  # we should use a more conservative gain.
1197  delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1198  newGain = 1 - abs(delta)
1199  # Average the gains to prevent oscillating solutions.
1200  newGain = (newGain + lastGain)/2.
1201  gain = max(baseGain, newGain)
1202  else:
1203  gain = baseGain
1204  gainList.append(gain)
1205  return gain
1206 
1207  def calculateModelWeights(self, dcrModels, dcrBBox):
1208  """Build an array that smoothly tapers to 0 away from detected sources.
1209 
1210  Parameters
1211  ----------
1212  dcrModels : `lsst.pipe.tasks.DcrModel`
1213  Best fit model of the true sky after correcting chromatic effects.
1214  dcrBBox : `lsst.geom.box.Box2I`
1215  Sub-region of the coadd which includes a buffer to allow for DCR.
1216 
1217  Returns
1218  -------
1219  weights : `numpy.ndarray` or `float`
1220  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1221  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1222 
1223  Raises
1224  ------
1225  ValueError
1226  If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1227  """
1228  if not self.config.useModelWeights:
1229  return 1.0
1230  if self.config.modelWeightsWidth < 0:
1231  raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1232  convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1233  convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1234  weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1235  weights[convergeMaskPixels] = 1.
1236  weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1237  weights /= np.max(weights)
1238  return weights
1239 
1240  def applyModelWeights(self, modelImages, refImage, modelWeights):
1241  """Smoothly replace model pixel values with those from a
1242  reference at locations away from detected sources.
1243 
1244  Parameters
1245  ----------
1246  modelImages : `list` of `lsst.afw.image.Image`
1247  The new DCR model images from the current iteration.
1248  The values will be modified in place.
1249  refImage : `lsst.afw.image.MaskedImage`
1250  A reference image used to supply the default pixel values.
1251  modelWeights : `numpy.ndarray` or `float`
1252  A 2D array of weight values that tapers smoothly to zero away from detected sources.
1253  Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1254  """
1255  if self.config.useModelWeights:
1256  for model in modelImages:
1257  model.array *= modelWeights
1258  model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1259 
1260  def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1261  """Pre-load sub-regions of a list of exposures.
1262 
1263  Parameters
1264  ----------
1265  bbox : `lsst.geom.box.Box2I`
1266  Sub-region to coadd
1267  statsCtrl : `lsst.afw.math.StatisticsControl`
1268  Statistics control object for coadd
1269  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1270  `lsst.daf.persistence.ButlerDataRef`
1271  The data references to the input warped exposures.
1272  imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1273  The image scalars correct for the zero point of the exposures.
1274  spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1275  Each element is dict with keys = mask plane name to add the spans to
1276 
1277  Returns
1278  -------
1279  subExposures : `dict`
1280  The `dict` keys are the visit IDs,
1281  and the values are `lsst.afw.image.ExposureF`
1282  The pre-loaded exposures for the current subregion.
1283  The variance plane contains weights, and not the variance
1284  """
1285  tempExpName = self.getTempExpDatasetName(self.warpType)
1286  zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1287  subExposures = {}
1288  for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1289  if isinstance(warpExpRef, DeferredDatasetHandle):
1290  exposure = warpExpRef.get(parameters={'bbox': bbox})
1291  else:
1292  exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1293  visit = warpExpRef.dataId["visit"]
1294  if altMaskSpans is not None:
1295  self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1296  imageScaler.scaleMaskedImage(exposure.maskedImage)
1297  # Note that the variance plane here is used to store weights, not the actual variance
1298  exposure.variance.array[:, :] = 0.
1299  # Set the weight of unmasked pixels to 1.
1300  exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1301  # Set the image value of masked pixels to zero.
1302  # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1303  exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1304  subExposures[visit] = exposure
1305  return subExposures
1306 
1307  def selectCoaddPsf(self, templateCoadd, warpRefList):
1308  """Compute the PSF of the coadd from the exposures with the best seeing.
1309 
1310  Parameters
1311  ----------
1312  templateCoadd : `lsst.afw.image.ExposureF`
1313  The initial coadd exposure before accounting for DCR.
1314  warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1315  `lsst.daf.persistence.ButlerDataRef`
1316  The data references to the input warped exposures.
1317 
1318  Returns
1319  -------
1320  psf : `lsst.meas.algorithms.CoaddPsf`
1321  The average PSF of the input exposures with the best seeing.
1322  """
1323  sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1324  tempExpName = self.getTempExpDatasetName(self.warpType)
1325  # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1326  # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1327  ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1328  psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1329  psfSizes = np.zeros(len(ccds))
1330  ccdVisits = np.array(ccds["visit"])
1331  for warpExpRef in warpRefList:
1332  if isinstance(warpExpRef, DeferredDatasetHandle):
1333  # Gen 3 API
1334  psf = warpExpRef.get(component="psf")
1335  else:
1336  # Gen 2 API. Delete this when Gen 2 retired
1337  psf = warpExpRef.get(tempExpName).getPsf()
1338  visit = warpExpRef.dataId["visit"]
1339  psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1340  psfSizes[ccdVisits == visit] = psfSize
1341  # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1342  # The selected PSFs are those that have a FWHM less than or equal to the smaller
1343  # of the mean or median FWHM of the input exposures.
1344  sizeThreshold = min(np.median(psfSizes), psfRefSize)
1345  goodPsfs = psfSizes <= sizeThreshold
1346  psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1347  self.config.coaddPsf.makeControl())
1348  return psf
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth)