Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23from math import ceil
24import numpy as np
25from scipy import ndimage
26import lsst.geom as geom
27import lsst.afw.image as afwImage
28import lsst.afw.table as afwTable
29import lsst.coadd.utils as coaddUtils
30from lsst.daf.butler import DeferredDatasetHandle
31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32import lsst.meas.algorithms as measAlg
33from lsst.meas.base import SingleFrameMeasurementTask
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.utils as utils
37from .assembleCoadd import (AssembleCoaddTask,
38 CompareWarpAssembleCoaddConfig,
39 CompareWarpAssembleCoaddTask)
40from .coaddBase import makeSkyInfo
41from .measurePsf import MeasurePsfTask
43__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
46class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("tract", "patch", "abstract_filter", "skymap"),
48 defaultTemplates={"inputCoaddName": "deep",
49 "outputCoaddName": "dcr",
50 "warpType": "direct",
51 "warpTypeSuffix": "",
52 "fakesType": ""}):
53 inputWarps = pipeBase.connectionTypes.Input(
54 doc=("Input list of warps to be assembled i.e. stacked."
55 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56 name="{inputCoaddName}Coadd_{warpType}Warp",
57 storageClass="ExposureF",
58 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59 deferLoad=True,
60 multiple=True
61 )
62 skyMap = pipeBase.connectionTypes.Input(
63 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64 name="{inputCoaddName}Coadd_skyMap",
65 storageClass="SkyMap",
66 dimensions=("skymap", ),
67 )
68 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70 " BRIGHT_OBJECT."),
71 name="brightObjectMask",
72 storageClass="ObjectMaskCatalog",
73 dimensions=("tract", "patch", "skymap", "abstract_filter"),
74 )
75 templateExposure = pipeBase.connectionTypes.Input(
76 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78 storageClass="ExposureF",
79 dimensions=("tract", "patch", "skymap", "abstract_filter"),
80 )
81 dcrCoadds = pipeBase.connectionTypes.Output(
82 doc="Output coadded exposure, produced by stacking input warps",
83 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84 storageClass="ExposureF",
85 dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
86 multiple=True,
87 )
88 dcrNImages = pipeBase.connectionTypes.Output(
89 doc="Output image of number of input images per pixel",
90 name="{outputCoaddName}Coadd_nImage",
91 storageClass="ImageU",
92 dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93 multiple=True,
94 )
96 def __init__(self, *, config=None):
97 super().__init__(config=config)
98 if not config.doWrite:
99 self.outputs.remove("dcrCoadds")
102class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103 pipelineConnections=DcrAssembleCoaddConnections):
104 dcrNumSubfilters = pexConfig.Field(
105 dtype=int,
106 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107 default=3,
108 )
109 maxNumIter = pexConfig.Field(
110 dtype=int,
111 optional=True,
112 doc="Maximum number of iterations of forward modeling.",
113 default=None,
114 )
115 minNumIter = pexConfig.Field(
116 dtype=int,
117 optional=True,
118 doc="Minimum number of iterations of forward modeling.",
119 default=None,
120 )
121 convergenceThreshold = pexConfig.Field(
122 dtype=float,
123 doc="Target relative change in convergence between iterations of forward modeling.",
124 default=0.001,
125 )
126 useConvergence = pexConfig.Field(
127 dtype=bool,
128 doc="Use convergence test as a forward modeling end condition?"
129 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130 default=True,
131 )
132 baseGain = pexConfig.Field(
133 dtype=float,
134 optional=True,
135 doc="Relative weight to give the new solution vs. the last solution when updating the model."
136 "A value of 1.0 gives equal weight to both solutions."
137 "Small values imply slower convergence of the solution, but can "
138 "help prevent overshooting and failures in the fit."
139 "If ``baseGain`` is None, a conservative gain "
140 "will be calculated from the number of subfilters. ",
141 default=None,
142 )
143 useProgressiveGain = pexConfig.Field(
144 dtype=bool,
145 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146 "When calculating the next gain, we use up to 5 previous gains and convergence values."
147 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148 default=True,
149 )
150 doAirmassWeight = pexConfig.Field(
151 dtype=bool,
152 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153 default=False,
154 )
155 modelWeightsWidth = pexConfig.Field(
156 dtype=float,
157 doc="Width of the region around detected sources to include in the DcrModel.",
158 default=3,
159 )
160 useModelWeights = pexConfig.Field(
161 dtype=bool,
162 doc="Width of the region around detected sources to include in the DcrModel.",
163 default=True,
164 )
165 splitSubfilters = pexConfig.Field(
166 dtype=bool,
167 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168 "Instead of at the midpoint",
169 default=True,
170 )
171 splitThreshold = pexConfig.Field(
172 dtype=float,
173 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174 "Set to 0 to always split the subfilters.",
175 default=0.1,
176 )
177 regularizeModelIterations = pexConfig.Field(
178 dtype=float,
179 doc="Maximum relative change of the model allowed between iterations."
180 "Set to zero to disable.",
181 default=2.,
182 )
183 regularizeModelFrequency = pexConfig.Field(
184 dtype=float,
185 doc="Maximum relative change of the model allowed between subfilters."
186 "Set to zero to disable.",
187 default=4.,
188 )
189 convergenceMaskPlanes = pexConfig.ListField(
190 dtype=str,
191 default=["DETECTED"],
192 doc="Mask planes to use to calculate convergence."
193 )
194 regularizationWidth = pexConfig.Field(
195 dtype=int,
196 default=2,
197 doc="Minimum radius of a region to include in regularization, in pixels."
198 )
199 imageInterpOrder = pexConfig.Field(
200 dtype=int,
201 doc="The order of the spline interpolation used to shift the image plane.",
202 default=3,
203 )
204 accelerateModel = pexConfig.Field(
205 dtype=float,
206 doc="Factor to amplify the differences between model planes by to speed convergence.",
207 default=3,
208 )
209 doCalculatePsf = pexConfig.Field(
210 dtype=bool,
211 doc="Set to detect stars and recalculate the PSF from the final coadd."
212 "Otherwise the PSF is estimated from a selection of the best input exposures",
213 default=False,
214 )
215 detectPsfSources = pexConfig.ConfigurableField(
216 target=measAlg.SourceDetectionTask,
217 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218 )
219 measurePsfSources = pexConfig.ConfigurableField(
220 target=SingleFrameMeasurementTask,
221 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222 )
223 measurePsf = pexConfig.ConfigurableField(
224 target=MeasurePsfTask,
225 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226 )
228 def setDefaults(self):
229 CompareWarpAssembleCoaddConfig.setDefaults(self)
230 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
231 self.doNImage = True
232 self.warpType = "direct"
233 self.assembleStaticSkyModel.warpType = self.warpType
234 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
235 self.assembleStaticSkyModel.doNImage = False
236 self.assembleStaticSkyModel.doWrite = False
237 self.detectPsfSources.returnOriginalFootprints = False
238 self.detectPsfSources.thresholdPolarity = "positive"
239 # Only use bright sources for PSF measurement
240 self.detectPsfSources.thresholdValue = 50
241 self.detectPsfSources.nSigmaToGrow = 2
242 # A valid star for PSF measurement should at least fill 5x5 pixels
243 self.detectPsfSources.minPixels = 25
244 # Use the variance plane to calculate signal to noise
245 self.detectPsfSources.thresholdType = "pixel_stdev"
246 # The signal to noise limit is good enough, while the flux limit is set
247 # in dimensionless units and may not be appropriate for all data sets.
248 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
251class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
252 """Assemble DCR coadded images from a set of warps.
254 Attributes
255 ----------
256 bufferSize : `int`
257 The number of pixels to grow each subregion by to allow for DCR.
259 Notes
260 -----
261 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
262 Warps (also called coadded temporary exposures), including the effects of
263 Differential Chromatic Refraction (DCR).
264 For full details of the mathematics and algorithm, please see
265 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
267 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
268 each subfilter used in the iterative calculation.
269 It begins by dividing the bandpass-defining filter into N equal bandwidth
270 "subfilters", and divides the flux in each pixel from an initial coadd
271 equally into each as a "dcrModel". Because the airmass and parallactic
272 angle of each individual exposure is known, we can calculate the shift
273 relative to the center of the band in each subfilter due to DCR. For each
274 exposure we apply this shift as a linear transformation to the dcrModels
275 and stack the results to produce a DCR-matched exposure. The matched
276 exposures are subtracted from the input exposures to produce a set of
277 residual images, and these residuals are reverse shifted for each
278 exposures' subfilters and stacked. The shifted and stacked residuals are
279 added to the dcrModels to produce a new estimate of the flux in each pixel
280 within each subfilter. The dcrModels are solved for iteratively, which
281 continues until the solution from a new iteration improves by less than
282 a set percentage, or a maximum number of iterations is reached.
283 Two forms of regularization are employed to reduce unphysical results.
284 First, the new solution is averaged with the solution from the previous
285 iteration, which mitigates oscillating solutions where the model
286 overshoots with alternating very high and low values.
287 Second, a common degeneracy when the data have a limited range of airmass or
288 parallactic angle values is for one subfilter to be fit with very low or
289 negative values, while another subfilter is fit with very high values. This
290 typically appears in the form of holes next to sources in one subfilter,
291 and corresponding extended wings in another. Because each subfilter has
292 a narrow bandwidth we assume that physical sources that are above the noise
293 level will not vary in flux by more than a factor of `frequencyClampFactor`
294 between subfilters, and pixels that have flux deviations larger than that
295 factor will have the excess flux distributed evenly among all subfilters.
296 If `splitSubfilters` is set, then each subfilter will be further sub-
297 divided during the forward modeling step (only). This approximates using
298 a higher number of subfilters that may be necessary for high airmass
299 observations, but does not increase the number of free parameters in the
300 fit. This is needed when there are high airmass observations which would
301 otherwise have significant DCR even within a subfilter. Because calculating
302 the shifted images takes most of the time, splitting the subfilters is
303 turned off by way of the `splitThreshold` option for low-airmass
304 observations that do not suffer from DCR within a subfilter.
305 """
307 ConfigClass = DcrAssembleCoaddConfig
308 _DefaultName = "dcrAssembleCoadd"
310 def __init__(self, *args, **kwargs):
311 super().__init__(*args, **kwargs)
312 if self.config.doCalculatePsf:
313 self.schema = afwTable.SourceTable.makeMinimalSchema()
314 self.makeSubtask("detectPsfSources", schema=self.schema)
315 self.makeSubtask("measurePsfSources", schema=self.schema)
316 self.makeSubtask("measurePsf", schema=self.schema)
318 @utils.inheritDoc(pipeBase.PipelineTask)
319 def runQuantum(self, butlerQC, inputRefs, outputRefs):
320 # Docstring to be formatted with info from PipelineTask.runQuantum
321 """
322 Notes
323 -----
324 Assemble a coadd from a set of Warps.
326 PipelineTask (Gen3) entry point to Coadd a set of Warps.
327 Analogous to `runDataRef`, it prepares all the data products to be
328 passed to `run`, and processes the results before returning a struct
329 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
330 Therefore, its inputs are accessed subregion by subregion
331 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
332 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
333 correspond to an update in `runDataRef` while both entry points
334 are used.
335 """
336 inputData = butlerQC.get(inputRefs)
338 # Construct skyInfo expected by run
339 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
340 skyMap = inputData["skyMap"]
341 outputDataId = butlerQC.quantum.dataId
343 inputData['skyInfo'] = makeSkyInfo(skyMap,
344 tractId=outputDataId['tract'],
345 patchId=outputDataId['patch'])
347 # Construct list of input Deferred Datasets
348 # These quack a bit like like Gen2 DataRefs
349 warpRefList = inputData['inputWarps']
350 # Perform same middle steps as `runDataRef` does
351 inputs = self.prepareInputs(warpRefList)
352 self.log.info("Found %d %s", len(inputs.tempExpRefList),
353 self.getTempExpDatasetName(self.warpType))
354 if len(inputs.tempExpRefList) == 0:
355 self.log.warn("No coadd temporary exposures found")
356 return
358 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
359 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
360 inputs.weightList, supplementaryData=supplementaryData)
362 inputData.setdefault('brightObjectMask', None)
363 for subfilter in range(self.config.dcrNumSubfilters):
364 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
365 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
366 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
368 if self.config.doWrite:
369 butlerQC.put(retStruct, outputRefs)
370 return retStruct
372 @pipeBase.timeMethod
373 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
374 """Assemble a coadd from a set of warps.
376 Coadd a set of Warps. Compute weights to be applied to each Warp and
377 find scalings to match the photometric zeropoint to a reference Warp.
378 Assemble the Warps using run method.
379 Forward model chromatic effects across multiple subfilters,
380 and subtract from the input Warps to build sets of residuals.
381 Use the residuals to construct a new ``DcrModel`` for each subfilter,
382 and iterate until the model converges.
383 Interpolate over NaNs and optionally write the coadd to disk.
384 Return the coadded exposure.
386 Parameters
387 ----------
388 dataRef : `lsst.daf.persistence.ButlerDataRef`
389 Data reference defining the patch for coaddition and the
390 reference Warp
391 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
392 List of data references to warps. Data to be coadded will be
393 selected from this list based on overlap with the patch defined by
394 the data reference.
396 Returns
397 -------
398 results : `lsst.pipe.base.Struct`
399 The Struct contains the following fields:
401 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
402 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
403 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
404 - ``dcrNImages``: `list` of exposure count images for each subfilter
405 """
406 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
407 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
409 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
410 warpRefList=warpRefList)
411 if results is None:
412 skyInfo = self.getSkyInfo(dataRef)
413 self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
414 skyInfo.patchInfo.getIndex())
415 return
416 if self.config.doCalculatePsf:
417 self.measureCoaddPsf(results.coaddExposure)
418 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
419 for subfilter in range(self.config.dcrNumSubfilters):
420 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
421 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
422 self.processResults(results.dcrCoadds[subfilter],
423 brightObjectMasks=brightObjects, dataId=dataRef.dataId)
424 if self.config.doWrite:
425 self.log.info("Persisting dcrCoadd")
426 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
427 numSubfilters=self.config.dcrNumSubfilters)
428 if self.config.doNImage and results.dcrNImages is not None:
429 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
430 numSubfilters=self.config.dcrNumSubfilters)
432 return results
434 @utils.inheritDoc(AssembleCoaddTask)
435 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
436 """Load the previously-generated template coadd.
438 This can be removed entirely once we no longer support the Gen 2 butler.
440 Returns
441 -------
442 templateCoadd : `lsst.pipe.base.Struct`
443 Result struct with components:
445 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
446 """
447 templateCoadd = butlerQC.get(inputRefs.templateExposure)
449 return pipeBase.Struct(templateCoadd=templateCoadd)
451 def measureCoaddPsf(self, coaddExposure):
452 """Detect sources on the coadd exposure and measure the final PSF.
454 Parameters
455 ----------
456 coaddExposure : `lsst.afw.image.Exposure`
457 The final coadded exposure.
458 """
459 table = afwTable.SourceTable.make(self.schema)
460 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
461 coaddSources = detResults.sources
462 self.measurePsfSources.run(
463 measCat=coaddSources,
464 exposure=coaddExposure
465 )
466 # Measure the PSF on the stacked subfilter coadds if possible.
467 # We should already have a decent estimate of the coadd PSF, however,
468 # so in case of any errors simply log them as a warning and use the
469 # default PSF.
470 try:
471 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
472 except Exception as e:
473 self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
474 else:
475 coaddExposure.setPsf(psfResults.psf)
477 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
478 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
480 Sets the property ``bufferSize``.
482 Parameters
483 ----------
484 templateCoadd : `lsst.afw.image.ExposureF`
485 The initial coadd exposure before accounting for DCR.
486 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
487 `lsst.daf.persistence.ButlerDataRef`
488 The data references to the input warped exposures.
489 weightList : `list` of `float`
490 The weight to give each input exposure in the coadd
491 Will be modified in place if ``doAirmassWeight`` is set.
493 Returns
494 -------
495 dcrModels : `lsst.pipe.tasks.DcrModel`
496 Best fit model of the true sky after correcting chromatic effects.
498 Raises
499 ------
500 NotImplementedError
501 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
502 """
503 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
504 filterInfo = templateCoadd.getFilter()
505 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
506 raise NotImplementedError("No minimum/maximum wavelength information found"
507 " in the filter definition! Please add lambdaMin and lambdaMax"
508 " to the Mapper class in your obs package.")
509 tempExpName = self.getTempExpDatasetName(self.warpType)
510 dcrShifts = []
511 airmassDict = {}
512 angleDict = {}
513 psfSizeDict = {}
514 for visitNum, warpExpRef in enumerate(warpRefList):
515 if isinstance(warpExpRef, DeferredDatasetHandle):
516 # Gen 3 API
517 visitInfo = warpExpRef.get(component="visitInfo")
518 psf = warpExpRef.get(component="psf")
519 visit = warpExpRef.datasetRefOrType.dataId["visit"]
520 else:
521 # Gen 2 API. Delete this when Gen 2 retired
522 visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
523 psf = warpExpRef.get(tempExpName).getPsf()
524 visit = warpExpRef.dataId["visit"]
525 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
526 airmass = visitInfo.getBoresightAirmass()
527 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
528 airmassDict[visit] = airmass
529 angleDict[visit] = parallacticAngle
530 psfSizeDict[visit] = psfSize
531 if self.config.doAirmassWeight:
532 weightList[visitNum] *= airmass
533 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
534 filterInfo, self.config.dcrNumSubfilters))))
535 self.log.info("Selected airmasses:\n%s", airmassDict)
536 self.log.info("Selected parallactic angles:\n%s", angleDict)
537 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
538 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
539 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
540 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
541 self.config.dcrNumSubfilters,
542 filterInfo=filterInfo,
543 psf=psf)
544 return dcrModels
546 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
547 supplementaryData=None):
548 """Assemble the coadd.
550 Requires additional inputs Struct ``supplementaryData`` to contain a
551 ``templateCoadd`` that serves as the model of the static sky.
553 Find artifacts and apply them to the warps' masks creating a list of
554 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
555 Then pass these alternative masks to the base class's assemble method.
557 Divide the ``templateCoadd`` evenly between each subfilter of a
558 ``DcrModel`` as the starting best estimate of the true wavelength-
559 dependent sky. Forward model the ``DcrModel`` using the known
560 chromatic effects in each subfilter and calculate a convergence metric
561 based on how well the modeled template matches the input warps. If
562 the convergence has not yet reached the desired threshold, then shift
563 and stack the residual images to build a new ``DcrModel``. Apply
564 conditioning to prevent oscillating solutions between iterations or
565 between subfilters.
567 Once the ``DcrModel`` reaches convergence or the maximum number of
568 iterations has been reached, fill the metadata for each subfilter
569 image and make them proper ``coaddExposure``s.
571 Parameters
572 ----------
573 skyInfo : `lsst.pipe.base.Struct`
574 Patch geometry information, from getSkyInfo
575 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
576 `lsst.daf.persistence.ButlerDataRef`
577 The data references to the input warped exposures.
578 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
579 The image scalars correct for the zero point of the exposures.
580 weightList : `list` of `float`
581 The weight to give each input exposure in the coadd
582 supplementaryData : `lsst.pipe.base.Struct`
583 Result struct returned by ``makeSupplementaryData`` with components:
585 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
587 Returns
588 -------
589 result : `lsst.pipe.base.Struct`
590 Result struct with components:
592 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
593 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
594 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
595 - ``dcrNImages``: `list` of exposure count images for each subfilter
596 """
597 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
598 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
599 templateCoadd = supplementaryData.templateCoadd
600 baseMask = templateCoadd.mask.clone()
601 # The variance plane is for each subfilter
602 # and should be proportionately lower than the full-band image
603 baseVariance = templateCoadd.variance.clone()
604 baseVariance /= self.config.dcrNumSubfilters
605 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
606 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
607 templateCoadd.setMask(baseMask)
608 badMaskPlanes = self.config.badMaskPlanes[:]
609 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
610 # This is because pixels in observations that are significantly affect by DCR
611 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
612 # but those are necessary to constrain the DCR model.
613 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
615 stats = self.prepareStats(mask=badPixelMask)
616 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
617 if self.config.doNImage:
618 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
619 spanSetMaskList, stats.ctrl)
620 nImage = afwImage.ImageU(skyInfo.bbox)
621 # Note that this nImage will be a factor of dcrNumSubfilters higher than
622 # the nImage returned by assembleCoadd for most pixels. This is because each
623 # subfilter may have a different nImage, and fractional values are not allowed.
624 for dcrNImage in dcrNImages:
625 nImage += dcrNImage
626 else:
627 dcrNImages = None
629 subregionSize = geom.Extent2I(*self.config.subregionSize)
630 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
631 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
632 subIter = 0
633 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
634 modelIter = 0
635 subIter += 1
636 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
637 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
638 dcrBBox = geom.Box2I(subBBox)
639 dcrBBox.grow(self.bufferSize)
640 dcrBBox.clip(dcrModels.bbox)
641 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
642 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
643 imageScalerList, spanSetMaskList)
644 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
645 warpRefList, weightList, stats.ctrl)
646 self.log.info("Initial convergence : %s", convergenceMetric)
647 convergenceList = [convergenceMetric]
648 gainList = []
649 convergenceCheck = 1.
650 refImage = templateCoadd.image
651 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
652 gain = self.calculateGain(convergenceList, gainList)
653 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
654 stats.ctrl, convergenceMetric, gain,
655 modelWeights, refImage, dcrWeights)
656 if self.config.useConvergence:
657 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
658 warpRefList, weightList, stats.ctrl)
659 if convergenceMetric == 0:
660 self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
661 "most likely due to there being no valid data in the region.",
662 skyInfo.patchInfo.getIndex(), subIter)
663 break
664 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
665 if (convergenceCheck < 0) & (modelIter > minNumIter):
666 self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
667 "iterations or desired convergence improvement of %s."
668 " Divergence: %s",
669 skyInfo.patchInfo.getIndex(), subIter,
670 self.config.convergenceThreshold, convergenceCheck)
671 break
672 convergenceList.append(convergenceMetric)
673 if modelIter > maxNumIter:
674 if self.config.useConvergence:
675 self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
676 "before reaching desired convergence improvement of %s."
677 " Final convergence improvement: %s",
678 skyInfo.patchInfo.getIndex(), subIter,
679 self.config.convergenceThreshold, convergenceCheck)
680 break
682 if self.config.useConvergence:
683 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
684 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
685 modelIter += 1
686 else:
687 if self.config.useConvergence:
688 self.log.info("Coadd patch %s subregion %s finished with "
689 "convergence metric %s after %s iterations",
690 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
691 else:
692 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
693 skyInfo.patchInfo.getIndex(), subIter, modelIter)
694 if self.config.useConvergence and convergenceMetric > 0:
695 self.log.info("Final convergence improvement was %.4f%% overall",
696 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
698 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
699 calibration=self.scaleZeroPoint.getPhotoCalib(),
700 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
701 mask=baseMask,
702 variance=baseVariance)
703 coaddExposure = self.stackCoadd(dcrCoadds)
704 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
705 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
707 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
708 """Calculate the number of exposures contributing to each subfilter.
710 Parameters
711 ----------
712 dcrModels : `lsst.pipe.tasks.DcrModel`
713 Best fit model of the true sky after correcting chromatic effects.
714 bbox : `lsst.geom.box.Box2I`
715 Bounding box of the patch to coadd.
716 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
717 `lsst.daf.persistence.ButlerDataRef`
718 The data references to the input warped exposures.
719 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
720 Each element of the `dict` contains the new mask plane name
721 (e.g. "CLIPPED and/or "NO_DATA") as the key,
722 and the list of SpanSets to apply to the mask.
723 statsCtrl : `lsst.afw.math.StatisticsControl`
724 Statistics control object for coadd
726 Returns
727 -------
728 dcrNImages : `list` of `lsst.afw.image.ImageU`
729 List of exposure count images for each subfilter
730 dcrWeights : `list` of `lsst.afw.image.ImageF`
731 Per-pixel weights for each subfilter.
732 Equal to 1/(number of unmasked images contributing to each pixel).
733 """
734 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
735 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
736 tempExpName = self.getTempExpDatasetName(self.warpType)
737 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
738 if isinstance(warpExpRef, DeferredDatasetHandle):
739 # Gen 3 API
740 exposure = warpExpRef.get(parameters={'bbox': bbox})
741 else:
742 # Gen 2 API. Delete this when Gen 2 retired
743 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
744 visitInfo = exposure.getInfo().getVisitInfo()
745 wcs = exposure.getInfo().getWcs()
746 mask = exposure.mask
747 if altMaskSpans is not None:
748 self.applyAltMaskPlanes(mask, altMaskSpans)
749 weightImage = np.zeros_like(exposure.image.array)
750 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
751 # The weights must be shifted in exactly the same way as the residuals,
752 # because they will be used as the denominator in the weighted average of residuals.
753 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
754 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
755 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
756 dcrWeight.array += shiftedWeights
757 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
758 weightsThreshold = 1.
759 goodPix = dcrWeights[0].array > weightsThreshold
760 for weights in dcrWeights[1:]:
761 goodPix = (weights.array > weightsThreshold) & goodPix
762 for subfilter in range(self.config.dcrNumSubfilters):
763 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
764 dcrWeights[subfilter].array[~goodPix] = 0.
765 dcrNImages[subfilter].array[~goodPix] = 0
766 return (dcrNImages, dcrWeights)
768 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
769 statsCtrl, convergenceMetric,
770 gain, modelWeights, refImage, dcrWeights):
771 """Assemble the DCR coadd for a sub-region.
773 Build a DCR-matched template for each input exposure, then shift the
774 residuals according to the DCR in each subfilter.
775 Stack the shifted residuals and apply them as a correction to the
776 solution from the previous iteration.
777 Restrict the new model solutions from varying by more than a factor of
778 `modelClampFactor` from the last solution, and additionally restrict the
779 individual subfilter models from varying by more than a factor of
780 `frequencyClampFactor` from their average.
781 Finally, mitigate potentially oscillating solutions by averaging the new
782 solution with the solution from the previous iteration, weighted by
783 their convergence metric.
785 Parameters
786 ----------
787 dcrModels : `lsst.pipe.tasks.DcrModel`
788 Best fit model of the true sky after correcting chromatic effects.
789 subExposures : `dict` of `lsst.afw.image.ExposureF`
790 The pre-loaded exposures for the current subregion.
791 bbox : `lsst.geom.box.Box2I`
792 Bounding box of the subregion to coadd.
793 dcrBBox : `lsst.geom.box.Box2I`
794 Sub-region of the coadd which includes a buffer to allow for DCR.
795 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
796 `lsst.daf.persistence.ButlerDataRef`
797 The data references to the input warped exposures.
798 statsCtrl : `lsst.afw.math.StatisticsControl`
799 Statistics control object for coadd
800 convergenceMetric : `float`
801 Quality of fit metric for the matched templates of the input images.
802 gain : `float`, optional
803 Relative weight to give the new solution when updating the model.
804 modelWeights : `numpy.ndarray` or `float`
805 A 2D array of weight values that tapers smoothly to zero away from detected sources.
806 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
807 refImage : `lsst.afw.image.Image`
808 A reference image used to supply the default pixel values.
809 dcrWeights : `list` of `lsst.afw.image.Image`
810 Per-pixel weights for each subfilter.
811 Equal to 1/(number of unmasked images contributing to each pixel).
812 """
813 residualGeneratorList = []
815 for warpExpRef in warpRefList:
816 if isinstance(warpExpRef, DeferredDatasetHandle):
817 visit = warpExpRef.datasetRefOrType.dataId["visit"]
818 else:
819 visit = warpExpRef.dataId["visit"]
820 exposure = subExposures[visit]
821 visitInfo = exposure.getInfo().getVisitInfo()
822 wcs = exposure.getInfo().getWcs()
823 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
824 order=self.config.imageInterpOrder,
825 splitSubfilters=self.config.splitSubfilters,
826 splitThreshold=self.config.splitThreshold,
827 amplifyModel=self.config.accelerateModel)
828 residual = exposure.image.array - templateImage.array
829 # Note that the variance plane here is used to store weights, not the actual variance
830 residual *= exposure.variance.array
831 # The residuals are stored as a list of generators.
832 # This allows the residual for a given subfilter and exposure to be created
833 # on the fly, instead of needing to store them all in memory.
834 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
836 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
837 gain=gain,
838 modelWeights=modelWeights,
839 refImage=refImage,
840 dcrWeights=dcrWeights)
841 dcrModels.assign(dcrSubModelOut, bbox)
843 def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
844 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
846 Parameters
847 ----------
848 residual : `numpy.ndarray`
849 The residual masked image for one exposure,
850 after subtracting the matched template
851 visitInfo : `lsst.afw.image.VisitInfo`
852 Metadata for the exposure.
853 wcs : `lsst.afw.geom.SkyWcs`
854 Coordinate system definition (wcs) for the exposure.
855 filterInfo : `lsst.afw.image.Filter`
856 The filter definition, set in the current instruments' obs package.
857 Required for any calculation of DCR, including making matched templates.
859 Yields
860 ------
861 residualImage : `numpy.ndarray`
862 The residual image for the next subfilter, shifted for DCR.
863 """
864 # Pre-calculate the spline-filtered residual image, so that step can be
865 # skipped in the shift calculation in `applyDcr`.
866 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
867 # Note that `splitSubfilters` is always turned off in the reverse direction.
868 # This option introduces additional blurring if applied to the residuals.
869 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
870 splitSubfilters=False)
871 for dcr in dcrShift:
872 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
873 doPrefilter=False, order=self.config.imageInterpOrder)
875 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
876 gain, modelWeights, refImage, dcrWeights):
877 """Calculate a new DcrModel from a set of image residuals.
879 Parameters
880 ----------
881 dcrModels : `lsst.pipe.tasks.DcrModel`
882 Current model of the true sky after correcting chromatic effects.
883 residualGeneratorList : `generator` of `numpy.ndarray`
884 The residual image for the next subfilter, shifted for DCR.
885 dcrBBox : `lsst.geom.box.Box2I`
886 Sub-region of the coadd which includes a buffer to allow for DCR.
887 statsCtrl : `lsst.afw.math.StatisticsControl`
888 Statistics control object for coadd
889 gain : `float`
890 Relative weight to give the new solution when updating the model.
891 modelWeights : `numpy.ndarray` or `float`
892 A 2D array of weight values that tapers smoothly to zero away from detected sources.
893 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
894 refImage : `lsst.afw.image.Image`
895 A reference image used to supply the default pixel values.
896 dcrWeights : `list` of `lsst.afw.image.Image`
897 Per-pixel weights for each subfilter.
898 Equal to 1/(number of unmasked images contributing to each pixel).
900 Returns
901 -------
902 dcrModel : `lsst.pipe.tasks.DcrModel`
903 New model of the true sky after correcting chromatic effects.
904 """
905 newModelImages = []
906 for subfilter, model in enumerate(dcrModels):
907 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
908 residual = np.sum(residualsList, axis=0)
909 residual *= dcrWeights[subfilter][dcrBBox].array
910 # `MaskedImage`s only support in-place addition, so rename for readability
911 newModel = model[dcrBBox].clone()
912 newModel.array += residual
913 # Catch any invalid values
914 badPixels = ~np.isfinite(newModel.array)
915 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
916 if self.config.regularizeModelIterations > 0:
917 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
918 self.config.regularizeModelIterations,
919 self.config.regularizationWidth)
920 newModelImages.append(newModel)
921 if self.config.regularizeModelFrequency > 0:
922 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
923 self.config.regularizeModelFrequency,
924 self.config.regularizationWidth)
925 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
926 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
927 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
928 dcrModels.mask, dcrModels.variance)
930 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
931 """Calculate a quality of fit metric for the matched templates.
933 Parameters
934 ----------
935 dcrModels : `lsst.pipe.tasks.DcrModel`
936 Best fit model of the true sky after correcting chromatic effects.
937 subExposures : `dict` of `lsst.afw.image.ExposureF`
938 The pre-loaded exposures for the current subregion.
939 bbox : `lsst.geom.box.Box2I`
940 Sub-region to coadd
941 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
942 `lsst.daf.persistence.ButlerDataRef`
943 The data references to the input warped exposures.
944 weightList : `list` of `float`
945 The weight to give each input exposure in the coadd
946 statsCtrl : `lsst.afw.math.StatisticsControl`
947 Statistics control object for coadd
949 Returns
950 -------
951 convergenceMetric : `float`
952 Quality of fit metric for all input exposures, within the sub-region
953 """
954 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
955 nSigma = 3.
956 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
957 bufferSize=self.bufferSize)
958 if np.max(significanceImage) == 0:
959 significanceImage += 1.
960 weight = 0
961 metric = 0.
962 metricList = {}
963 for warpExpRef, expWeight in zip(warpRefList, weightList):
964 if isinstance(warpExpRef, DeferredDatasetHandle):
965 visit = warpExpRef.datasetRefOrType.dataId["visit"]
966 else:
967 visit = warpExpRef.dataId["visit"]
968 exposure = subExposures[visit][bbox]
969 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
970 metric += singleMetric
971 metricList[visit] = singleMetric
972 weight += 1.
973 self.log.info("Individual metrics:\n%s", metricList)
974 return 1.0 if weight == 0.0 else metric/weight
976 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
977 """Calculate a quality of fit metric for a single matched template.
979 Parameters
980 ----------
981 dcrModels : `lsst.pipe.tasks.DcrModel`
982 Best fit model of the true sky after correcting chromatic effects.
983 exposure : `lsst.afw.image.ExposureF`
984 The input warped exposure to evaluate.
985 significanceImage : `numpy.ndarray`
986 Array of weights for each pixel corresponding to its significance
987 for the convergence calculation.
988 statsCtrl : `lsst.afw.math.StatisticsControl`
989 Statistics control object for coadd
991 Returns
992 -------
993 convergenceMetric : `float`
994 Quality of fit metric for one exposure, within the sub-region.
995 """
996 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
997 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
998 order=self.config.imageInterpOrder,
999 splitSubfilters=self.config.splitSubfilters,
1000 splitThreshold=self.config.splitThreshold,
1001 amplifyModel=self.config.accelerateModel)
1002 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1003 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1005 finitePixels = np.isfinite(diffVals)
1006 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1007 convergeMaskPixels = exposure.mask.array & convergeMask > 0
1008 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1009 if np.sum(usePixels) == 0:
1010 metric = 0.
1011 else:
1012 diffUse = diffVals[usePixels]
1013 refUse = refVals[usePixels]
1014 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1015 return metric
1017 def stackCoadd(self, dcrCoadds):
1018 """Add a list of sub-band coadds together.
1020 Parameters
1021 ----------
1022 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1023 A list of coadd exposures, each exposure containing
1024 the model for one subfilter.
1026 Returns
1027 -------
1028 coaddExposure : `lsst.afw.image.ExposureF`
1029 A single coadd exposure that is the sum of the sub-bands.
1030 """
1031 coaddExposure = dcrCoadds[0].clone()
1032 for coadd in dcrCoadds[1:]:
1033 coaddExposure.maskedImage += coadd.maskedImage
1034 return coaddExposure
1036 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1037 mask=None, variance=None):
1038 """Create a list of coadd exposures from a list of masked images.
1040 Parameters
1041 ----------
1042 dcrModels : `lsst.pipe.tasks.DcrModel`
1043 Best fit model of the true sky after correcting chromatic effects.
1044 skyInfo : `lsst.pipe.base.Struct`
1045 Patch geometry information, from getSkyInfo
1046 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1047 `lsst.daf.persistence.ButlerDataRef`
1048 The data references to the input warped exposures.
1049 weightList : `list` of `float`
1050 The weight to give each input exposure in the coadd
1051 calibration : `lsst.afw.Image.PhotoCalib`, optional
1052 Scale factor to set the photometric calibration of an exposure.
1053 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1054 A record of the observations that are included in the coadd.
1055 mask : `lsst.afw.image.Mask`, optional
1056 Optional mask to override the values in the final coadd.
1057 variance : `lsst.afw.image.Image`, optional
1058 Optional variance plane to override the values in the final coadd.
1060 Returns
1061 -------
1062 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1063 A list of coadd exposures, each exposure containing
1064 the model for one subfilter.
1065 """
1066 dcrCoadds = []
1067 refModel = dcrModels.getReferenceImage()
1068 for model in dcrModels:
1069 if self.config.accelerateModel > 1:
1070 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1071 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1072 if calibration is not None:
1073 coaddExposure.setPhotoCalib(calibration)
1074 if coaddInputs is not None:
1075 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1076 # Set the metadata for the coadd, including PSF and aperture corrections.
1077 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1078 # Overwrite the PSF
1079 coaddExposure.setPsf(dcrModels.psf)
1080 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1081 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1082 maskedImage.image = model
1083 maskedImage.mask = dcrModels.mask
1084 maskedImage.variance = dcrModels.variance
1085 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1086 if mask is not None:
1087 coaddExposure.setMask(mask)
1088 if variance is not None:
1089 coaddExposure.setVariance(variance)
1090 dcrCoadds.append(coaddExposure)
1091 return dcrCoadds
1093 def calculateGain(self, convergenceList, gainList):
1094 """Calculate the gain to use for the current iteration.
1096 After calculating a new DcrModel, each value is averaged with the
1097 value in the corresponding pixel from the previous iteration. This
1098 reduces oscillating solutions that iterative techniques are plagued by,
1099 and speeds convergence. By far the biggest changes to the model
1100 happen in the first couple iterations, so we can also use a more
1101 aggressive gain later when the model is changing slowly.
1103 Parameters
1104 ----------
1105 convergenceList : `list` of `float`
1106 The quality of fit metric from each previous iteration.
1107 gainList : `list` of `float`
1108 The gains used in each previous iteration: appended with the new
1109 gain value.
1110 Gains are numbers between ``self.config.baseGain`` and 1.
1112 Returns
1113 -------
1114 gain : `float`
1115 Relative weight to give the new solution when updating the model.
1116 A value of 1.0 gives equal weight to both solutions.
1118 Raises
1119 ------
1120 ValueError
1121 If ``len(convergenceList) != len(gainList)+1``.
1122 """
1123 nIter = len(convergenceList)
1124 if nIter != len(gainList) + 1:
1125 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1126 % (len(convergenceList), len(gainList)))
1128 if self.config.baseGain is None:
1129 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1130 # The more subfilters being modeled, the lower the gain should be.
1131 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1132 else:
1133 baseGain = self.config.baseGain
1135 if self.config.useProgressiveGain and nIter > 2:
1136 # To calculate the best gain to use, compare the past gains that have been used
1137 # with the resulting convergences to estimate the best gain to use.
1138 # Algorithmically, this is a Kalman filter.
1139 # If forward modeling proceeds perfectly, the convergence metric should
1140 # asymptotically approach a final value.
1141 # We can estimate that value from the measured changes in convergence
1142 # weighted by the gains used in each previous iteration.
1143 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1144 for i in range(nIter - 1)]
1145 # The convergence metric is strictly positive, so if the estimated final convergence is
1146 # less than zero, force it to zero.
1147 estFinalConv = np.array(estFinalConv)
1148 estFinalConv[estFinalConv < 0] = 0
1149 # Because the estimate may slowly change over time, only use the most recent measurements.
1150 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1151 lastGain = gainList[-1]
1152 lastConv = convergenceList[-2]
1153 newConv = convergenceList[-1]
1154 # The predicted convergence is the value we would get if the new model calculated
1155 # in the previous iteration was perfect. Recall that the updated model that is
1156 # actually used is the gain-weighted average of the new and old model,
1157 # so the convergence would be similarly weighted.
1158 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1159 # If the measured and predicted convergence are very close, that indicates
1160 # that our forward model is accurate and we can use a more aggressive gain
1161 # If the measured convergence is significantly worse (or better!) than predicted,
1162 # that indicates that the model is not converging as expected and
1163 # we should use a more conservative gain.
1164 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1165 newGain = 1 - abs(delta)
1166 # Average the gains to prevent oscillating solutions.
1167 newGain = (newGain + lastGain)/2.
1168 gain = max(baseGain, newGain)
1169 else:
1170 gain = baseGain
1171 gainList.append(gain)
1172 return gain
1174 def calculateModelWeights(self, dcrModels, dcrBBox):
1175 """Build an array that smoothly tapers to 0 away from detected sources.
1177 Parameters
1178 ----------
1179 dcrModels : `lsst.pipe.tasks.DcrModel`
1180 Best fit model of the true sky after correcting chromatic effects.
1181 dcrBBox : `lsst.geom.box.Box2I`
1182 Sub-region of the coadd which includes a buffer to allow for DCR.
1184 Returns
1185 -------
1186 weights : `numpy.ndarray` or `float`
1187 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1188 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1190 Raises
1191 ------
1192 ValueError
1193 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1194 """
1195 if not self.config.useModelWeights:
1196 return 1.0
1197 if self.config.modelWeightsWidth < 0:
1198 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1199 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1200 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1201 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1202 weights[convergeMaskPixels] = 1.
1203 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1204 weights /= np.max(weights)
1205 return weights
1207 def applyModelWeights(self, modelImages, refImage, modelWeights):
1208 """Smoothly replace model pixel values with those from a
1209 reference at locations away from detected sources.
1211 Parameters
1212 ----------
1213 modelImages : `list` of `lsst.afw.image.Image`
1214 The new DCR model images from the current iteration.
1215 The values will be modified in place.
1216 refImage : `lsst.afw.image.MaskedImage`
1217 A reference image used to supply the default pixel values.
1218 modelWeights : `numpy.ndarray` or `float`
1219 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1220 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1221 """
1222 if self.config.useModelWeights:
1223 for model in modelImages:
1224 model.array *= modelWeights
1225 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1227 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1228 """Pre-load sub-regions of a list of exposures.
1230 Parameters
1231 ----------
1232 bbox : `lsst.geom.box.Box2I`
1233 Sub-region to coadd
1234 statsCtrl : `lsst.afw.math.StatisticsControl`
1235 Statistics control object for coadd
1236 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1237 `lsst.daf.persistence.ButlerDataRef`
1238 The data references to the input warped exposures.
1239 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1240 The image scalars correct for the zero point of the exposures.
1241 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1242 Each element is dict with keys = mask plane name to add the spans to
1244 Returns
1245 -------
1246 subExposures : `dict`
1247 The `dict` keys are the visit IDs,
1248 and the values are `lsst.afw.image.ExposureF`
1249 The pre-loaded exposures for the current subregion.
1250 The variance plane contains weights, and not the variance
1251 """
1252 tempExpName = self.getTempExpDatasetName(self.warpType)
1253 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1254 subExposures = {}
1255 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1256 if isinstance(warpExpRef, DeferredDatasetHandle):
1257 visit = warpExpRef.datasetRefOrType.dataId["visit"]
1258 exposure = warpExpRef.get(parameters={'bbox': bbox})
1259 else:
1260 visit = warpExpRef.dataId["visit"]
1261 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1262 if altMaskSpans is not None:
1263 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1264 imageScaler.scaleMaskedImage(exposure.maskedImage)
1265 # Note that the variance plane here is used to store weights, not the actual variance
1266 exposure.variance.array[:, :] = 0.
1267 # Set the weight of unmasked pixels to 1.
1268 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1269 # Set the image value of masked pixels to zero.
1270 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1271 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1272 subExposures[visit] = exposure
1273 return subExposures
1275 def selectCoaddPsf(self, templateCoadd, warpRefList):
1276 """Compute the PSF of the coadd from the exposures with the best seeing.
1278 Parameters
1279 ----------
1280 templateCoadd : `lsst.afw.image.ExposureF`
1281 The initial coadd exposure before accounting for DCR.
1282 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1283 `lsst.daf.persistence.ButlerDataRef`
1284 The data references to the input warped exposures.
1286 Returns
1287 -------
1288 psf : `lsst.meas.algorithms.CoaddPsf`
1289 The average PSF of the input exposures with the best seeing.
1290 """
1291 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1292 tempExpName = self.getTempExpDatasetName(self.warpType)
1293 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1294 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1295 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1296 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1297 psfSizes = np.zeros(len(ccds))
1298 ccdVisits = np.array(ccds["visit"])
1299 for warpExpRef in warpRefList:
1300 if isinstance(warpExpRef, DeferredDatasetHandle):
1301 # Gen 3 API
1302 psf = warpExpRef.get(component="psf")
1303 visit = warpExpRef.datasetRefOrType.dataId["visit"]
1304 else:
1305 # Gen 2 API. Delete this when Gen 2 retired
1306 psf = warpExpRef.get(tempExpName).getPsf()
1307 visit = warpExpRef.dataId["visit"]
1308 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1309 psfSizes[ccdVisits == visit] = psfSize
1310 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1311 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1312 # of the mean or median FWHM of the input exposures.
1313 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1314 goodPsfs = psfSizes <= sizeThreshold
1315 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1316 self.config.coaddPsf.makeControl())
1317 return psf