Coverage for python/lsst/drp/tasks/dcr_assemble_coadd.py: 15%
413 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-16 13:25 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-16 13:25 +0000
1# This file is part of drp_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
24from math import ceil
26import lsst.afw.image as afwImage
27import lsst.afw.table as afwTable
28import lsst.coadd.utils as coaddUtils
29import lsst.geom as geom
30import lsst.meas.algorithms as measAlg
31import lsst.pex.config as pexConfig
32import lsst.pipe.base as pipeBase
33import lsst.utils as utils
34import numpy as np
35from lsst.ip.diffim.dcrModel import DcrModel, applyDcr, calculateDcr
36from lsst.meas.base import SingleFrameMeasurementTask
37from lsst.pipe.tasks.coaddBase import makeSkyInfo, subBBoxIter
38from lsst.pipe.tasks.measurePsf import MeasurePsfTask
39from lsst.utils.timer import timeMethod
40from scipy import ndimage
42from .assemble_coadd import (
43 AssembleCoaddConnections,
44 AssembleCoaddTask,
45 CompareWarpAssembleCoaddConfig,
46 CompareWarpAssembleCoaddTask,
47)
50class DcrAssembleCoaddConnections(
51 AssembleCoaddConnections,
52 dimensions=("tract", "patch", "band", "skymap"),
53 defaultTemplates={
54 "inputWarpName": "deep",
55 "inputCoaddName": "deep",
56 "outputCoaddName": "dcr",
57 "warpType": "direct",
58 "warpTypeSuffix": "",
59 "fakesType": "",
60 },
61):
62 inputWarps = pipeBase.connectionTypes.Input(
63 doc=(
64 "Input list of warps to be assembled i.e. stacked."
65 "Note that this will often be different than the inputCoaddName."
66 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"
67 ),
68 name="{inputWarpName}Coadd_{warpType}Warp",
69 storageClass="ExposureF",
70 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
71 deferLoad=True,
72 multiple=True,
73 )
74 templateExposure = pipeBase.connectionTypes.Input(
75 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
76 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
77 storageClass="ExposureF",
78 dimensions=("tract", "patch", "skymap", "band"),
79 )
80 dcrCoadds = pipeBase.connectionTypes.Output(
81 doc="Output coadded exposure, produced by stacking input warps",
82 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
83 storageClass="ExposureF",
84 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
85 multiple=True,
86 )
87 dcrNImages = pipeBase.connectionTypes.Output(
88 doc="Output image of number of input images per pixel",
89 name="{outputCoaddName}Coadd_nImage",
90 storageClass="ImageU",
91 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
92 multiple=True,
93 )
95 def __init__(self, *, config=None):
96 super().__init__(config=config)
97 if not config.doWrite:
98 self.outputs.remove("dcrCoadds")
99 if not config.doNImage:
100 self.outputs.remove("dcrNImages")
101 # Remove outputs inherited from ``AssembleCoaddConnections`` that are
102 # not used.
103 self.outputs.remove("coaddExposure")
104 self.outputs.remove("nImage")
107class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig, pipelineConnections=DcrAssembleCoaddConnections):
108 dcrNumSubfilters = pexConfig.Field(
109 dtype=int,
110 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
111 default=3,
112 )
113 maxNumIter = pexConfig.Field(
114 dtype=int,
115 optional=True,
116 doc="Maximum number of iterations of forward modeling.",
117 default=None,
118 )
119 minNumIter = pexConfig.Field(
120 dtype=int,
121 optional=True,
122 doc="Minimum number of iterations of forward modeling.",
123 default=None,
124 )
125 convergenceThreshold = pexConfig.Field(
126 dtype=float,
127 doc="Target relative change in convergence between iterations of forward modeling.",
128 default=0.001,
129 )
130 useConvergence = pexConfig.Field(
131 dtype=bool,
132 doc="Use convergence test as a forward modeling end condition?"
133 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
134 default=True,
135 )
136 baseGain = pexConfig.Field(
137 dtype=float,
138 optional=True,
139 doc="Relative weight to give the new solution vs. the last solution when updating the model."
140 "A value of 1.0 gives equal weight to both solutions."
141 "Small values imply slower convergence of the solution, but can "
142 "help prevent overshooting and failures in the fit."
143 "If ``baseGain`` is None, a conservative gain "
144 "will be calculated from the number of subfilters. ",
145 default=None,
146 )
147 useProgressiveGain = pexConfig.Field(
148 dtype=bool,
149 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
150 "When calculating the next gain, we use up to 5 previous gains and convergence values."
151 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
152 default=True,
153 )
154 doAirmassWeight = pexConfig.Field(
155 dtype=bool,
156 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
157 default=False,
158 )
159 modelWeightsWidth = pexConfig.Field(
160 dtype=float,
161 doc="Width of the region around detected sources to include in the DcrModel.",
162 default=3,
163 )
164 useModelWeights = pexConfig.Field(
165 dtype=bool,
166 doc="Width of the region around detected sources to include in the DcrModel.",
167 default=True,
168 )
169 splitSubfilters = pexConfig.Field(
170 dtype=bool,
171 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter. Instead of at the midpoint",
172 default=True,
173 )
174 splitThreshold = pexConfig.Field(
175 dtype=float,
176 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
177 "Set to 0 to always split the subfilters.",
178 default=0.1,
179 )
180 regularizeModelIterations = pexConfig.Field(
181 dtype=float,
182 doc="Maximum relative change of the model allowed between iterations. Set to zero to disable.",
183 default=2.0,
184 )
185 regularizeModelFrequency = pexConfig.Field(
186 dtype=float,
187 doc="Maximum relative change of the model allowed between subfilters. Set to zero to disable.",
188 default=4.0,
189 )
190 convergenceMaskPlanes = pexConfig.ListField(
191 dtype=str, default=["DETECTED"], doc="Mask planes to use to calculate convergence."
192 )
193 regularizationWidth = pexConfig.Field(
194 dtype=int, default=2, doc="Minimum radius of a region to include in regularization, in pixels."
195 )
196 imageInterpOrder = pexConfig.Field(
197 dtype=int,
198 doc="The order of the spline interpolation used to shift the image plane.",
199 default=1,
200 )
201 accelerateModel = pexConfig.Field(
202 dtype=float,
203 doc="Factor to amplify the differences between model planes by to speed convergence.",
204 default=3,
205 )
206 doCalculatePsf = pexConfig.Field(
207 dtype=bool,
208 doc="Set to detect stars and recalculate the PSF from the final coadd."
209 "Otherwise the PSF is estimated from a selection of the best input exposures",
210 default=False,
211 )
212 detectPsfSources = pexConfig.ConfigurableField(
213 target=measAlg.SourceDetectionTask,
214 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
215 )
216 measurePsfSources = pexConfig.ConfigurableField(
217 target=SingleFrameMeasurementTask,
218 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set.",
219 )
220 measurePsf = pexConfig.ConfigurableField(
221 target=MeasurePsfTask,
222 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
223 )
224 effectiveWavelength = pexConfig.Field(
225 doc="Effective wavelength of the filter, in nm."
226 "Required if transmission curves aren't used."
227 "Support for using transmission curves is to be added in DM-13668.",
228 dtype=float,
229 )
230 bandwidth = pexConfig.Field(
231 doc="Bandwidth of the physical filter, in nm."
232 "Required if transmission curves aren't used."
233 "Support for using transmission curves is to be added in DM-13668.",
234 dtype=float,
235 )
237 def setDefaults(self):
238 CompareWarpAssembleCoaddConfig.setDefaults(self)
239 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
240 self.doNImage = True
241 self.assembleStaticSkyModel.warpType = self.warpType
242 # The deepCoadd and nImage files will be overwritten by this Task, so
243 # don't write them the first time.
244 self.assembleStaticSkyModel.doNImage = False
245 self.assembleStaticSkyModel.doWrite = False
246 self.detectPsfSources.returnOriginalFootprints = False
247 self.detectPsfSources.thresholdPolarity = "positive"
248 # Only use bright sources for PSF measurement
249 self.detectPsfSources.thresholdValue = 50
250 self.detectPsfSources.nSigmaToGrow = 2
251 # A valid star for PSF measurement should at least fill 5x5 pixels
252 self.detectPsfSources.minPixels = 25
253 # Use the variance plane to calculate signal to noise
254 self.detectPsfSources.thresholdType = "pixel_stdev"
255 # Ensure psf candidate size is as large as piff psf size.
256 if (
257 self.doCalculatePsf
258 and self.measurePsf.psfDeterminer.name == "piff"
259 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize
260 ):
261 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize
264class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
265 """Assemble DCR coadded images from a set of warps.
267 Attributes
268 ----------
269 bufferSize : `int`
270 The number of pixels to grow each subregion by to allow for DCR.
272 Notes
273 -----
274 As with AssembleCoaddTask, we want to assemble a coadded image from a set
275 of Warps (also called coadded temporary exposures), including the effects
276 of Differential Chromatic Refraction (DCR).
277 For full details of the mathematics and algorithm, please see
278 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
280 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
281 each subfilter used in the iterative calculation.
282 It begins by dividing the bandpass-defining filter into N equal bandwidth
283 "subfilters", and divides the flux in each pixel from an initial coadd
284 equally into each as a "dcrModel". Because the airmass and parallactic
285 angle of each individual exposure is known, we can calculate the shift
286 relative to the center of the band in each subfilter due to DCR. For each
287 exposure we apply this shift as a linear transformation to the dcrModels
288 and stack the results to produce a DCR-matched exposure. The matched
289 exposures are subtracted from the input exposures to produce a set of
290 residual images, and these residuals are reverse shifted for each
291 exposures' subfilters and stacked. The shifted and stacked residuals are
292 added to the dcrModels to produce a new estimate of the flux in each pixel
293 within each subfilter. The dcrModels are solved for iteratively, which
294 continues until the solution from a new iteration improves by less than
295 a set percentage, or a maximum number of iterations is reached.
296 Two forms of regularization are employed to reduce unphysical results.
297 First, the new solution is averaged with the solution from the previous
298 iteration, which mitigates oscillating solutions where the model
299 overshoots with alternating very high and low values.
300 Second, a common degeneracy when the data have a limited range of airmass
301 or parallactic angle values is for one subfilter to be fit with very low or
302 negative values, while another subfilter is fit with very high values. This
303 typically appears in the form of holes next to sources in one subfilter,
304 and corresponding extended wings in another. Because each subfilter has
305 a narrow bandwidth we assume that physical sources that are above the noise
306 level will not vary in flux by more than a factor of `frequencyClampFactor`
307 between subfilters, and pixels that have flux deviations larger than that
308 factor will have the excess flux distributed evenly among all subfilters.
309 If `splitSubfilters` is set, then each subfilter will be further sub-
310 divided during the forward modeling step (only). This approximates using
311 a higher number of subfilters that may be necessary for high airmass
312 observations, but does not increase the number of free parameters in the
313 fit. This is needed when there are high airmass observations which would
314 otherwise have significant DCR even within a subfilter. Because calculating
315 the shifted images takes most of the time, splitting the subfilters is
316 turned off by way of the `splitThreshold` option for low-airmass
317 observations that do not suffer from DCR within a subfilter.
318 """
320 ConfigClass = DcrAssembleCoaddConfig
321 _DefaultName = "dcrAssembleCoadd"
323 def __init__(self, *args, **kwargs):
324 super().__init__(*args, **kwargs)
325 if self.config.doCalculatePsf:
326 self.schema = afwTable.SourceTable.makeMinimalSchema()
327 self.makeSubtask("detectPsfSources", schema=self.schema)
328 self.makeSubtask("measurePsfSources", schema=self.schema)
329 self.makeSubtask("measurePsf", schema=self.schema)
331 @utils.inheritDoc(pipeBase.PipelineTask)
332 def runQuantum(self, butlerQC, inputRefs, outputRefs):
333 # Docstring to be formatted with info from PipelineTask.runQuantum
334 """
335 Notes
336 -----
337 Assemble a coadd from a set of Warps.
338 """
339 inputData = butlerQC.get(inputRefs)
341 # Construct skyInfo expected by run
342 # Do not remove skyMap from inputData in case _makeSupplementaryData
343 # needs it.
344 skyMap = inputData["skyMap"]
345 outputDataId = butlerQC.quantum.dataId
347 inputData["skyInfo"] = makeSkyInfo(
348 skyMap, tractId=outputDataId["tract"], patchId=outputDataId["patch"]
349 )
351 # Construct list of input Deferred Datasets
352 warpRefList = inputData["inputWarps"]
354 inputs = self.prepareInputs(warpRefList)
355 self.log.info("Found %d %s", len(inputs.tempExpRefList), self.getTempExpDatasetName(self.warpType))
356 if len(inputs.tempExpRefList) == 0:
357 self.log.warning("No coadd temporary exposures found")
358 return
360 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
361 retStruct = self.run(
362 inputData["skyInfo"],
363 inputs.tempExpRefList,
364 inputs.imageScalerList,
365 inputs.weightList,
366 supplementaryData=supplementaryData,
367 )
369 inputData.setdefault("brightObjectMask", None)
370 for subfilter in range(self.config.dcrNumSubfilters):
371 # Use the PSF of the stacked dcrModel, and do not recalculate the
372 # PSF for each subfilter
373 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
374 self.processResults(retStruct.dcrCoadds[subfilter], inputData["brightObjectMask"], outputDataId)
376 if self.config.doWrite:
377 butlerQC.put(retStruct, outputRefs)
378 return retStruct
380 @utils.inheritDoc(AssembleCoaddTask)
381 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
382 """Load the previously-generated template coadd.
384 Returns
385 -------
386 templateCoadd : `lsst.pipe.base.Struct`
387 Results as a struct with attributes:
389 ``templateCoadd``
390 Coadded exposure (`lsst.afw.image.ExposureF`).
391 """
392 templateCoadd = butlerQC.get(inputRefs.templateExposure)
394 return pipeBase.Struct(templateCoadd=templateCoadd)
396 def measureCoaddPsf(self, coaddExposure):
397 """Detect sources on the coadd exposure and measure the final PSF.
399 Parameters
400 ----------
401 coaddExposure : `lsst.afw.image.Exposure`
402 The final coadded exposure.
403 """
404 table = afwTable.SourceTable.make(self.schema)
405 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
406 coaddSources = detResults.sources
407 self.measurePsfSources.run(measCat=coaddSources, exposure=coaddExposure)
408 # Measure the PSF on the stacked subfilter coadds if possible.
409 # We should already have a decent estimate of the coadd PSF, however,
410 # so in case of any errors simply log them as a warning and use the
411 # default PSF.
412 try:
413 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
414 except Exception as e:
415 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e)
416 else:
417 coaddExposure.setPsf(psfResults.psf)
419 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
420 """Prepare the DCR coadd by iterating through the visitInfo of the
421 input warps.
423 Sets the property ``bufferSize``.
425 Parameters
426 ----------
427 templateCoadd : `lsst.afw.image.ExposureF`
428 The initial coadd exposure before accounting for DCR.
429 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
430 The data references to the input warped exposures.
431 weightList : `list` of `float`
432 The weight to give each input exposure in the coadd.
433 Will be modified in place if ``doAirmassWeight`` is set.
435 Returns
436 -------
437 dcrModels : `lsst.pipe.tasks.DcrModel`
438 Best fit model of the true sky after correcting chromatic effects.
440 Raises
441 ------
442 NotImplementedError
443 If ``lambdaMin`` is missing from the Mapper class of the obs
444 package being used.
445 """
446 sigma2fwhm = 2.0 * np.sqrt(2.0 * np.log(2.0))
447 filterLabel = templateCoadd.getFilter()
448 dcrShifts = []
449 airmassDict = {}
450 angleDict = {}
451 psfSizeDict = {}
452 for visitNum, warpExpRef in enumerate(warpRefList):
453 visitInfo = warpExpRef.get(component="visitInfo")
454 psf = warpExpRef.get(component="psf")
455 visit = warpExpRef.dataId["visit"]
456 # Just need a rough estimate; average positions are fine
457 psfAvgPos = psf.getAveragePosition()
458 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius() * sigma2fwhm
459 airmass = visitInfo.getBoresightAirmass()
460 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
461 airmassDict[visit] = airmass
462 angleDict[visit] = parallacticAngle
463 psfSizeDict[visit] = psfSize
464 if self.config.doAirmassWeight:
465 weightList[visitNum] *= airmass
466 dcrShifts.append(
467 np.max(
468 np.abs(
469 calculateDcr(
470 visitInfo,
471 templateCoadd.getWcs(),
472 self.config.effectiveWavelength,
473 self.config.bandwidth,
474 self.config.dcrNumSubfilters,
475 )
476 )
477 )
478 )
479 self.log.info("Selected airmasses:\n%s", airmassDict)
480 self.log.info("Selected parallactic angles:\n%s", angleDict)
481 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
482 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
483 try:
484 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
485 except Exception as e:
486 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e)
487 else:
488 psf = templateCoadd.getPsf()
489 dcrModels = DcrModel.fromImage(
490 templateCoadd.maskedImage,
491 self.config.dcrNumSubfilters,
492 effectiveWavelength=self.config.effectiveWavelength,
493 bandwidth=self.config.bandwidth,
494 wcs=templateCoadd.getWcs(),
495 filterLabel=filterLabel,
496 psf=psf,
497 )
498 return dcrModels
500 @timeMethod
501 def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None):
502 r"""Assemble the coadd.
504 Requires additional inputs Struct ``supplementaryData`` to contain a
505 ``templateCoadd`` that serves as the model of the static sky.
507 Find artifacts and apply them to the warps' masks creating a list of
508 alternative masks with a new "CLIPPED" plane and updated "NO_DATA"
509 plane then pass these alternative masks to the base class's assemble
510 method.
512 Divide the ``templateCoadd`` evenly between each subfilter of a
513 ``DcrModel`` as the starting best estimate of the true wavelength-
514 dependent sky. Forward model the ``DcrModel`` using the known
515 chromatic effects in each subfilter and calculate a convergence metric
516 based on how well the modeled template matches the input warps. If
517 the convergence has not yet reached the desired threshold, then shift
518 and stack the residual images to build a new ``DcrModel``. Apply
519 conditioning to prevent oscillating solutions between iterations or
520 between subfilters.
522 Once the ``DcrModel`` reaches convergence or the maximum number of
523 iterations has been reached, fill the metadata for each subfilter
524 image and make them proper ``coaddExposure``\ s.
526 Parameters
527 ----------
528 skyInfo : `lsst.pipe.base.Struct`
529 Patch geometry information, from getSkyInfo
530 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
531 The data references to the input warped exposures.
532 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
533 The image scalars correct for the zero point of the exposures.
534 weightList : `list` of `float`
535 The weight to give each input exposure in the coadd
536 supplementaryData : `lsst.pipe.base.Struct`
537 Result struct returned by ``_makeSupplementaryData`` with
538 attributes:
540 ``templateCoadd``
541 Coadded exposure (`lsst.afw.image.Exposure`).
543 Returns
544 -------
545 result : `lsst.pipe.base.Struct`
546 Results as a struct with attributes:
548 ``coaddExposure``
549 Coadded exposure (`lsst.afw.image.Exposure`).
550 ``nImage``
551 Exposure count image (`lsst.afw.image.ImageU`).
552 ``dcrCoadds``
553 `list` of coadded exposures for each subfilter.
554 ``dcrNImages``
555 `list` of exposure count images for each subfilter.
556 """
557 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
558 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters * 3
559 templateCoadd = supplementaryData.templateCoadd
560 baseMask = templateCoadd.mask.clone()
561 # The variance plane is for each subfilter
562 # and should be proportionately lower than the full-band image
563 baseVariance = templateCoadd.variance.clone()
564 baseVariance /= self.config.dcrNumSubfilters
565 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
566 # Note that the mask gets cleared in ``findArtifacts``, but we want to
567 # preserve the mask.
568 templateCoadd.setMask(baseMask)
569 badMaskPlanes = self.config.badMaskPlanes[:]
570 # Note that is important that we do not add "CLIPPED" to
571 # ``badMaskPlanes``. This is because pixels in observations that are
572 # significantly affected by DCR are likely to have many pixels that are
573 # both "DETECTED" and "CLIPPED", but those are necessary to constrain
574 # the DCR model.
575 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
577 stats = self.prepareStats(mask=badPixelMask)
578 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
579 if self.config.doNImage:
580 dcrNImages, dcrWeights = self.calculateNImage(
581 dcrModels, skyInfo.bbox, warpRefList, spanSetMaskList, stats.ctrl
582 )
583 nImage = afwImage.ImageU(skyInfo.bbox)
584 # Note that this nImage will be a factor of dcrNumSubfilters higher
585 # than the nImage returned by assembleCoadd for most pixels. This
586 # is because each subfilter may have a different nImage, and
587 # fractional values are not allowed.
588 for dcrNImage in dcrNImages:
589 nImage += dcrNImage
590 else:
591 dcrNImages = None
593 subregionSize = geom.Extent2I(*self.config.subregionSize)
594 nSubregions = ceil(skyInfo.bbox.getHeight() / subregionSize[1]) * ceil(
595 skyInfo.bbox.getWidth() / subregionSize[0]
596 )
597 subIter = 0
598 for subBBox in subBBoxIter(skyInfo.bbox, subregionSize):
599 modelIter = 0
600 subIter += 1
601 self.log.info(
602 "Computing coadd over patch %s subregion %s of %s: %s",
603 skyInfo.patchInfo.getIndex(),
604 subIter,
605 nSubregions,
606 subBBox,
607 )
608 dcrBBox = geom.Box2I(subBBox)
609 dcrBBox.grow(self.bufferSize)
610 dcrBBox.clip(dcrModels.bbox)
611 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
612 subExposures = self.loadSubExposures(
613 dcrBBox, stats.ctrl, warpRefList, imageScalerList, spanSetMaskList
614 )
615 convergenceMetric = self.calculateConvergence(
616 dcrModels, subExposures, subBBox, warpRefList, weightList, stats.ctrl
617 )
618 self.log.info("Initial convergence : %s", convergenceMetric)
619 convergenceList = [convergenceMetric]
620 gainList = []
621 convergenceCheck = 1.0
622 refImage = templateCoadd.image
623 while convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter:
624 gain = self.calculateGain(convergenceList, gainList)
625 self.dcrAssembleSubregion(
626 dcrModels,
627 subExposures,
628 subBBox,
629 dcrBBox,
630 warpRefList,
631 stats.ctrl,
632 convergenceMetric,
633 gain,
634 modelWeights,
635 refImage,
636 dcrWeights,
637 )
638 if self.config.useConvergence:
639 convergenceMetric = self.calculateConvergence(
640 dcrModels, subExposures, subBBox, warpRefList, weightList, stats.ctrl
641 )
642 if convergenceMetric == 0:
643 self.log.warning(
644 "Coadd patch %s subregion %s had convergence metric of 0.0 which is "
645 "most likely due to there being no valid data in the region.",
646 skyInfo.patchInfo.getIndex(),
647 subIter,
648 )
649 break
650 convergenceCheck = (convergenceList[-1] - convergenceMetric) / convergenceMetric
651 if (convergenceCheck < 0) & (modelIter > minNumIter):
652 self.log.warning(
653 "Coadd patch %s subregion %s diverged before reaching maximum "
654 "iterations or desired convergence improvement of %s."
655 " Divergence: %s",
656 skyInfo.patchInfo.getIndex(),
657 subIter,
658 self.config.convergenceThreshold,
659 convergenceCheck,
660 )
661 break
662 convergenceList.append(convergenceMetric)
663 if modelIter > maxNumIter:
664 if self.config.useConvergence:
665 self.log.warning(
666 "Coadd patch %s subregion %s reached maximum iterations "
667 "before reaching desired convergence improvement of %s."
668 " Final convergence improvement: %s",
669 skyInfo.patchInfo.getIndex(),
670 subIter,
671 self.config.convergenceThreshold,
672 convergenceCheck,
673 )
674 break
676 if self.config.useConvergence:
677 self.log.info(
678 "Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
679 modelIter,
680 convergenceMetric,
681 100.0 * convergenceCheck,
682 gain,
683 )
684 modelIter += 1
685 else:
686 if self.config.useConvergence:
687 self.log.info(
688 "Coadd patch %s subregion %s finished with "
689 "convergence metric %s after %s iterations",
690 skyInfo.patchInfo.getIndex(),
691 subIter,
692 convergenceMetric,
693 modelIter,
694 )
695 else:
696 self.log.info(
697 "Coadd patch %s subregion %s finished after %s iterations",
698 skyInfo.patchInfo.getIndex(),
699 subIter,
700 modelIter,
701 )
702 if self.config.useConvergence and convergenceMetric > 0:
703 self.log.info(
704 "Final convergence improvement was %.4f%% overall",
705 100 * (convergenceList[0] - convergenceMetric) / convergenceMetric,
706 )
708 dcrCoadds = self.fillCoadd(
709 dcrModels,
710 skyInfo,
711 warpRefList,
712 weightList,
713 calibration=self.scaleZeroPoint.getPhotoCalib(),
714 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
715 mask=baseMask,
716 variance=baseVariance,
717 )
718 coaddExposure = self.stackCoadd(dcrCoadds)
719 return pipeBase.Struct(
720 coaddExposure=coaddExposure, nImage=nImage, dcrCoadds=dcrCoadds, dcrNImages=dcrNImages
721 )
723 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
724 """Calculate the number of exposures contributing to each subfilter.
726 Parameters
727 ----------
728 dcrModels : `lsst.pipe.tasks.DcrModel`
729 Best fit model of the true sky after correcting chromatic effects.
730 bbox : `lsst.geom.box.Box2I`
731 Bounding box of the patch to coadd.
732 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
733 The data references to the input warped exposures.
734 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None`
735 Each element of the `dict` contains the new mask plane name
736 (e.g. "CLIPPED and/or "NO_DATA") as the key,
737 and the list of SpanSets to apply to the mask.
738 statsCtrl : `lsst.afw.math.StatisticsControl`
739 Statistics control object for coadd
741 Returns
742 -------
743 dcrNImages : `list` of `lsst.afw.image.ImageU`
744 List of exposure count images for each subfilter.
745 dcrWeights : `list` of `lsst.afw.image.ImageF`
746 Per-pixel weights for each subfilter.
747 Equal to 1/(number of unmasked images contributing to each pixel).
748 """
749 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
750 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
751 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
752 exposure = warpExpRef.get(parameters={"bbox": bbox})
753 visitInfo = exposure.getInfo().getVisitInfo()
754 wcs = exposure.getInfo().getWcs()
755 mask = exposure.mask
756 if altMaskSpans is not None:
757 self.applyAltMaskPlanes(mask, altMaskSpans)
758 weightImage = np.zeros_like(exposure.image.array)
759 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.0
760 # The weights must be shifted in exactly the same way as the
761 # residuals, because they will be used as the denominator in the
762 # weighted average of residuals.
763 weightsGenerator = self.dcrResiduals(
764 weightImage, visitInfo, wcs, dcrModels.effectiveWavelength, dcrModels.bandwidth
765 )
766 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
767 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
768 dcrWeight.array += shiftedWeights
769 # Exclude any pixels that don't have at least one exposure contributing
770 # in all subfilters
771 weightsThreshold = 1.0
772 goodPix = dcrWeights[0].array > weightsThreshold
773 for weights in dcrWeights[1:]:
774 goodPix = (weights.array > weightsThreshold) & goodPix
775 for subfilter in range(self.config.dcrNumSubfilters):
776 dcrWeights[subfilter].array[goodPix] = 1.0 / dcrWeights[subfilter].array[goodPix]
777 dcrWeights[subfilter].array[~goodPix] = 0.0
778 dcrNImages[subfilter].array[~goodPix] = 0
779 return (dcrNImages, dcrWeights)
781 def dcrAssembleSubregion(
782 self,
783 dcrModels,
784 subExposures,
785 bbox,
786 dcrBBox,
787 warpRefList,
788 statsCtrl,
789 convergenceMetric,
790 gain,
791 modelWeights,
792 refImage,
793 dcrWeights,
794 ):
795 """Assemble the DCR coadd for a sub-region.
797 Build a DCR-matched template for each input exposure, then shift the
798 residuals according to the DCR in each subfilter.
799 Stack the shifted residuals and apply them as a correction to the
800 solution from the previous iteration.
801 Restrict the new model solutions from varying by more than a factor of
802 `modelClampFactor` from the last solution, and additionally restrict
803 the individual subfilter models from varying by more than a factor of
804 `frequencyClampFactor` from their average.
805 Finally, mitigate potentially oscillating solutions by averaging the
806 new solution with the solution from the previous iteration, weighted by
807 their convergence metric.
809 Parameters
810 ----------
811 dcrModels : `lsst.pipe.tasks.DcrModel`
812 Best fit model of the true sky after correcting chromatic effects.
813 subExposures : `dict` of `lsst.afw.image.ExposureF`
814 The pre-loaded exposures for the current subregion.
815 bbox : `lsst.geom.box.Box2I`
816 Bounding box of the subregion to coadd.
817 dcrBBox : `lsst.geom.box.Box2I`
818 Sub-region of the coadd which includes a buffer to allow for DCR.
819 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
820 The data references to the input warped exposures.
821 statsCtrl : `lsst.afw.math.StatisticsControl`
822 Statistics control object for coadd.
823 convergenceMetric : `float`
824 Quality of fit metric for the matched templates of the input
825 images.
826 gain : `float`, optional
827 Relative weight to give the new solution when updating the model.
828 modelWeights : `numpy.ndarray` or `float`
829 A 2D array of weight values that tapers smoothly to zero away from
830 detected sources. Set to a placeholder value of 1.0 if
831 ``self.config.useModelWeights`` is False.
832 refImage : `lsst.afw.image.Image`
833 A reference image used to supply the default pixel values.
834 dcrWeights : `list` of `lsst.afw.image.Image`
835 Per-pixel weights for each subfilter.
836 Equal to 1/(number of unmasked images contributing to each pixel).
837 """
838 residualGeneratorList = []
840 for warpExpRef in warpRefList:
841 visit = warpExpRef.dataId["visit"]
842 exposure = subExposures[visit]
843 visitInfo = exposure.getInfo().getVisitInfo()
844 wcs = exposure.getInfo().getWcs()
845 templateImage = dcrModels.buildMatchedTemplate(
846 exposure=exposure,
847 bbox=exposure.getBBox(),
848 order=self.config.imageInterpOrder,
849 splitSubfilters=self.config.splitSubfilters,
850 splitThreshold=self.config.splitThreshold,
851 amplifyModel=self.config.accelerateModel,
852 )
853 residual = exposure.image.array - templateImage.array
854 # Note that the variance plane here is used to store weights, not
855 # the actual variance
856 residual *= exposure.variance.array
857 # The residuals are stored as a list of generators.
858 # This allows the residual for a given subfilter and exposure to be
859 # created on the fly, instead of needing to store them all in
860 # memory.
861 residualGeneratorList.append(
862 self.dcrResiduals(
863 residual, visitInfo, wcs, dcrModels.effectiveWavelength, dcrModels.bandwidth
864 )
865 )
867 dcrSubModelOut = self.newModelFromResidual(
868 dcrModels,
869 residualGeneratorList,
870 dcrBBox,
871 statsCtrl,
872 gain=gain,
873 modelWeights=modelWeights,
874 refImage=refImage,
875 dcrWeights=dcrWeights,
876 )
877 dcrModels.assign(dcrSubModelOut, bbox)
879 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
880 """Prepare a residual image for stacking in each subfilter by applying
881 the reverse DCR shifts.
883 Parameters
884 ----------
885 residual : `numpy.ndarray`
886 The residual masked image for one exposure,
887 after subtracting the matched template.
888 visitInfo : `lsst.afw.image.VisitInfo`
889 Metadata for the exposure.
890 wcs : `lsst.afw.geom.SkyWcs`
891 Coordinate system definition (wcs) for the exposure.
893 Yields
894 ------
895 residualImage : `numpy.ndarray`
896 The residual image for the next subfilter, shifted for DCR.
897 """
898 if self.config.imageInterpOrder > 1:
899 # Pre-calculate the spline-filtered residual image, so that step
900 # can be skipped in the shift calculation in `applyDcr`.
901 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
902 else:
903 # No need to prefilter if order=1 (it will also raise an error)
904 filteredResidual = residual
905 # Note that `splitSubfilters` is always turned off in the reverse
906 # direction. This option introduces additional blurring if applied to
907 # the residuals.
908 dcrShift = calculateDcr(
909 visitInfo,
910 wcs,
911 effectiveWavelength,
912 bandwidth,
913 self.config.dcrNumSubfilters,
914 splitSubfilters=False,
915 )
916 for dcr in dcrShift:
917 yield applyDcr(
918 filteredResidual,
919 dcr,
920 useInverse=True,
921 splitSubfilters=False,
922 doPrefilter=False,
923 order=self.config.imageInterpOrder,
924 )
926 def newModelFromResidual(
927 self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights
928 ):
929 """Calculate a new DcrModel from a set of image residuals.
931 Parameters
932 ----------
933 dcrModels : `lsst.pipe.tasks.DcrModel`
934 Current model of the true sky after correcting chromatic effects.
935 residualGeneratorList : `generator` of `numpy.ndarray`
936 The residual image for the next subfilter, shifted for DCR.
937 dcrBBox : `lsst.geom.box.Box2I`
938 Sub-region of the coadd which includes a buffer to allow for DCR.
939 statsCtrl : `lsst.afw.math.StatisticsControl`
940 Statistics control object for coadd.
941 gain : `float`
942 Relative weight to give the new solution when updating the model.
943 modelWeights : `numpy.ndarray` or `float`
944 A 2D array of weight values that tapers smoothly to zero away from
945 detected sources. Set to a placeholder value of 1.0 if
946 ``self.config.useModelWeights`` is False.
947 refImage : `lsst.afw.image.Image`
948 A reference image used to supply the default pixel values.
949 dcrWeights : `list` of `lsst.afw.image.Image`
950 Per-pixel weights for each subfilter.
951 Equal to 1/(number of unmasked images contributing to each pixel).
953 Returns
954 -------
955 dcrModel : `lsst.pipe.tasks.DcrModel`
956 New model of the true sky after correcting chromatic effects.
957 """
958 newModelImages = []
959 for subfilter, model in enumerate(dcrModels):
960 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
961 residual = np.sum(residualsList, axis=0)
962 residual *= dcrWeights[subfilter][dcrBBox].array
963 # `MaskedImage`s only support in-place addition, so rename for
964 # readability.
965 newModel = model[dcrBBox].clone()
966 newModel.array += residual
967 # Catch any invalid values
968 badPixels = ~np.isfinite(newModel.array)
969 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
970 if self.config.regularizeModelIterations > 0:
971 dcrModels.regularizeModelIter(
972 subfilter,
973 newModel,
974 dcrBBox,
975 self.config.regularizeModelIterations,
976 self.config.regularizationWidth,
977 )
978 newModelImages.append(newModel)
979 if self.config.regularizeModelFrequency > 0:
980 dcrModels.regularizeModelFreq(
981 newModelImages,
982 dcrBBox,
983 statsCtrl,
984 self.config.regularizeModelFrequency,
985 self.config.regularizationWidth,
986 )
987 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
988 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
989 return DcrModel(
990 newModelImages,
991 dcrModels.filter,
992 dcrModels.effectiveWavelength,
993 dcrModels.bandwidth,
994 dcrModels.psf,
995 dcrModels.mask,
996 dcrModels.variance,
997 )
999 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
1000 """Calculate a quality of fit metric for the matched templates.
1002 Parameters
1003 ----------
1004 dcrModels : `lsst.pipe.tasks.DcrModel`
1005 Best fit model of the true sky after correcting chromatic effects.
1006 subExposures : `dict` of `lsst.afw.image.ExposureF`
1007 The pre-loaded exposures for the current subregion.
1008 bbox : `lsst.geom.box.Box2I`
1009 Sub-region to coadd.
1010 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1011 The data references to the input warped exposures.
1012 weightList : `list` of `float`
1013 The weight to give each input exposure in the coadd.
1014 statsCtrl : `lsst.afw.math.StatisticsControl`
1015 Statistics control object for coadd.
1017 Returns
1018 -------
1019 convergenceMetric : `float`
1020 Quality of fit metric for all input exposures, within the
1021 sub-region.
1022 """
1023 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
1024 nSigma = 3.0
1025 significanceImage += nSigma * dcrModels.calculateNoiseCutoff(
1026 dcrModels[1], statsCtrl, bufferSize=self.bufferSize
1027 )
1028 if np.max(significanceImage) == 0:
1029 significanceImage += 1.0
1030 weight = 0
1031 metric = 0.0
1032 metricList = {}
1033 for warpExpRef, expWeight in zip(warpRefList, weightList):
1034 visit = warpExpRef.dataId["visit"]
1035 exposure = subExposures[visit][bbox]
1036 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
1037 metric += singleMetric
1038 metricList[visit] = singleMetric
1039 weight += 1.0
1040 self.log.info("Individual metrics:\n%s", metricList)
1041 return 1.0 if weight == 0.0 else metric / weight
1043 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
1044 """Calculate a quality of fit metric for a single matched template.
1046 Parameters
1047 ----------
1048 dcrModels : `lsst.pipe.tasks.DcrModel`
1049 Best fit model of the true sky after correcting chromatic effects.
1050 exposure : `lsst.afw.image.ExposureF`
1051 The input warped exposure to evaluate.
1052 significanceImage : `numpy.ndarray`
1053 Array of weights for each pixel corresponding to its significance
1054 for the convergence calculation.
1055 statsCtrl : `lsst.afw.math.StatisticsControl`
1056 Statistics control object for coadd.
1058 Returns
1059 -------
1060 convergenceMetric : `float`
1061 Quality of fit metric for one exposure, within the sub-region.
1062 """
1063 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1064 templateImage = dcrModels.buildMatchedTemplate(
1065 exposure=exposure,
1066 bbox=exposure.getBBox(),
1067 order=self.config.imageInterpOrder,
1068 splitSubfilters=self.config.splitSubfilters,
1069 splitThreshold=self.config.splitThreshold,
1070 amplifyModel=self.config.accelerateModel,
1071 )
1072 diffVals = np.abs(exposure.image.array - templateImage.array) * significanceImage
1073 refVals = np.abs(exposure.image.array + templateImage.array) * significanceImage / 2.0
1075 finitePixels = np.isfinite(diffVals)
1076 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1077 convergeMaskPixels = exposure.mask.array & convergeMask > 0
1078 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1079 if np.sum(usePixels) == 0:
1080 metric = 0.0
1081 else:
1082 diffUse = diffVals[usePixels]
1083 refUse = refVals[usePixels]
1084 metric = np.sum(diffUse / np.median(diffUse)) / np.sum(refUse / np.median(diffUse))
1085 return metric
1087 def stackCoadd(self, dcrCoadds):
1088 """Add a list of sub-band coadds together.
1090 Parameters
1091 ----------
1092 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1093 A list of coadd exposures, each exposure containing
1094 the model for one subfilter.
1096 Returns
1097 -------
1098 coaddExposure : `lsst.afw.image.ExposureF`
1099 A single coadd exposure that is the sum of the sub-bands.
1100 """
1101 coaddExposure = dcrCoadds[0].clone()
1102 for coadd in dcrCoadds[1:]:
1103 coaddExposure.maskedImage += coadd.maskedImage
1104 return coaddExposure
1106 def fillCoadd(
1107 self,
1108 dcrModels,
1109 skyInfo,
1110 warpRefList,
1111 weightList,
1112 calibration=None,
1113 coaddInputs=None,
1114 mask=None,
1115 variance=None,
1116 ):
1117 """Create a list of coadd exposures from a list of masked images.
1119 Parameters
1120 ----------
1121 dcrModels : `lsst.pipe.tasks.DcrModel`
1122 Best fit model of the true sky after correcting chromatic effects.
1123 skyInfo : `lsst.pipe.base.Struct`
1124 Patch geometry information, from getSkyInfo.
1125 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1126 The data references to the input warped exposures.
1127 weightList : `list` of `float`
1128 The weight to give each input exposure in the coadd.
1129 calibration : `lsst.afw.Image.PhotoCalib`, optional
1130 Scale factor to set the photometric calibration of an exposure.
1131 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1132 A record of the observations that are included in the coadd.
1133 mask : `lsst.afw.image.Mask`, optional
1134 Optional mask to override the values in the final coadd.
1135 variance : `lsst.afw.image.Image`, optional
1136 Optional variance plane to override the values in the final coadd.
1138 Returns
1139 -------
1140 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1141 A list of coadd exposures, each exposure containing
1142 the model for one subfilter.
1143 """
1144 dcrCoadds = []
1145 refModel = dcrModels.getReferenceImage()
1146 for model in dcrModels:
1147 if self.config.accelerateModel > 1:
1148 model.array = (model.array - refModel) * self.config.accelerateModel + refModel
1149 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1150 if calibration is not None:
1151 coaddExposure.setPhotoCalib(calibration)
1152 if coaddInputs is not None:
1153 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1154 # Set the metadata for the coadd, including PSF and aperture
1155 # corrections.
1156 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1157 # Overwrite the PSF
1158 coaddExposure.setPsf(dcrModels.psf)
1159 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1160 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1161 maskedImage.image = model
1162 maskedImage.mask = dcrModels.mask
1163 maskedImage.variance = dcrModels.variance
1164 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1165 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1166 if mask is not None:
1167 coaddExposure.setMask(mask)
1168 if variance is not None:
1169 coaddExposure.setVariance(variance)
1170 dcrCoadds.append(coaddExposure)
1171 return dcrCoadds
1173 def calculateGain(self, convergenceList, gainList):
1174 """Calculate the gain to use for the current iteration.
1176 After calculating a new DcrModel, each value is averaged with the
1177 value in the corresponding pixel from the previous iteration. This
1178 reduces oscillating solutions that iterative techniques are plagued by,
1179 and speeds convergence. By far the biggest changes to the model
1180 happen in the first couple iterations, so we can also use a more
1181 aggressive gain later when the model is changing slowly.
1183 Parameters
1184 ----------
1185 convergenceList : `list` of `float`
1186 The quality of fit metric from each previous iteration.
1187 gainList : `list` of `float`
1188 The gains used in each previous iteration: appended with the new
1189 gain value.
1190 Gains are numbers between ``self.config.baseGain`` and 1.
1192 Returns
1193 -------
1194 gain : `float`
1195 Relative weight to give the new solution when updating the model.
1196 A value of 1.0 gives equal weight to both solutions.
1198 Raises
1199 ------
1200 ValueError
1201 If ``len(convergenceList) != len(gainList)+1``.
1202 """
1203 nIter = len(convergenceList)
1204 if nIter != len(gainList) + 1:
1205 raise ValueError(
1206 "convergenceList (%d) must be one element longer than gainList (%d)."
1207 % (len(convergenceList), len(gainList))
1208 )
1210 if self.config.baseGain is None:
1211 # If ``baseGain`` is not set, calculate it from the number of DCR
1212 # subfilters. The more subfilters being modeled, the lower the gain
1213 # should be.
1214 baseGain = 1.0 / (self.config.dcrNumSubfilters - 1)
1215 else:
1216 baseGain = self.config.baseGain
1218 if self.config.useProgressiveGain and nIter > 2:
1219 # To calculate the best gain to use, compare the past gains that
1220 # have been used with the resulting convergences to estimate the
1221 # best gain to use. Algorithmically, this is a Kalman filter.
1222 # If forward modeling proceeds perfectly, the convergence metric
1223 # should asymptotically approach a final value. We can estimate
1224 # that value from the measured changes in convergence weighted by
1225 # the gains used in each previous iteration.
1226 estFinalConv = [
1227 ((1 + gainList[i]) * convergenceList[i + 1] - convergenceList[i]) / gainList[i]
1228 for i in range(nIter - 1)
1229 ]
1230 # The convergence metric is strictly positive, so if the estimated
1231 # final convergence is less than zero, force it to zero.
1232 estFinalConv = np.array(estFinalConv)
1233 estFinalConv[estFinalConv < 0] = 0
1234 # Because the estimate may slowly change over time, only use the
1235 # most recent measurements.
1236 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0) :])
1237 lastGain = gainList[-1]
1238 lastConv = convergenceList[-2]
1239 newConv = convergenceList[-1]
1240 # The predicted convergence is the value we would get if the new
1241 # model calculated in the previous iteration was perfect. Recall
1242 # that the updated model that is actually used is the gain-weighted
1243 # average of the new and old model, so the convergence would be
1244 # similarly weighted.
1245 predictedConv = (estFinalConv * lastGain + lastConv) / (1.0 + lastGain)
1246 # If the measured and predicted convergence are very close, that
1247 # indicates that our forward model is accurate and we can use a
1248 # more aggressive gain. If the measured convergence is
1249 # significantly worse (or better!) than predicted, that indicates
1250 # that the model is not converging as expected and we should use a
1251 # more conservative gain.
1252 delta = (predictedConv - newConv) / ((lastConv - estFinalConv) / (1 + lastGain))
1253 newGain = 1 - abs(delta)
1254 # Average the gains to prevent oscillating solutions.
1255 newGain = (newGain + lastGain) / 2.0
1256 gain = max(baseGain, newGain)
1257 else:
1258 gain = baseGain
1259 gainList.append(gain)
1260 return gain
1262 def calculateModelWeights(self, dcrModels, dcrBBox):
1263 """Build an array that smoothly tapers to 0 away from detected sources.
1265 Parameters
1266 ----------
1267 dcrModels : `lsst.pipe.tasks.DcrModel`
1268 Best fit model of the true sky after correcting chromatic effects.
1269 dcrBBox : `lsst.geom.box.Box2I`
1270 Sub-region of the coadd which includes a buffer to allow for DCR.
1272 Returns
1273 -------
1274 weights : `numpy.ndarray` or `float`
1275 A 2D array of weight values that tapers smoothly to zero away from
1276 detected sources. Set to a placeholder value of 1.0 if
1277 ``self.config.useModelWeights`` is False.
1279 Raises
1280 ------
1281 ValueError
1282 If ``useModelWeights`` is set and ``modelWeightsWidth`` is
1283 negative.
1284 """
1285 if not self.config.useModelWeights:
1286 return 1.0
1287 if self.config.modelWeightsWidth < 0:
1288 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1289 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1290 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1291 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1292 weights[convergeMaskPixels] = 1.0
1293 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth)
1294 weights /= np.max(weights)
1295 return weights
1297 def applyModelWeights(self, modelImages, refImage, modelWeights):
1298 """Smoothly replace model pixel values with those from a
1299 reference at locations away from detected sources.
1301 Parameters
1302 ----------
1303 modelImages : `list` of `lsst.afw.image.Image`
1304 The new DCR model images from the current iteration.
1305 The values will be modified in place.
1306 refImage : `lsst.afw.image.MaskedImage`
1307 A reference image used to supply the default pixel values.
1308 modelWeights : `numpy.ndarray` or `float`
1309 A 2D array of weight values that tapers smoothly to zero away from
1310 detected sources. Set to a placeholder value of 1.0 if
1311 ``self.config.useModelWeights`` is False.
1312 """
1313 if self.config.useModelWeights:
1314 for model in modelImages:
1315 model.array *= modelWeights
1316 model.array += refImage.array * (1.0 - modelWeights) / self.config.dcrNumSubfilters
1318 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1319 """Pre-load sub-regions of a list of exposures.
1321 Parameters
1322 ----------
1323 bbox : `lsst.geom.box.Box2I`
1324 Sub-region to coadd.
1325 statsCtrl : `lsst.afw.math.StatisticsControl`
1326 Statistics control object for coadd.
1327 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1328 The data references to the input warped exposures.
1329 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1330 The image scalars correct for the zero point of the exposures.
1331 spanSetMaskList : `list` of `dict` containing spanSet lists, or `None`
1332 Each element is dict with keys = mask plane name to add the spans
1333 to.
1335 Returns
1336 -------
1337 subExposures : `dict`
1338 The `dict` keys are the visit IDs,
1339 and the values are `lsst.afw.image.ExposureF`
1340 The pre-loaded exposures for the current subregion.
1341 The variance plane contains weights, and not the variance
1342 """
1343 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1344 subExposures = {}
1345 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1346 exposure = warpExpRef.get(parameters={"bbox": bbox})
1347 visit = warpExpRef.dataId["visit"]
1348 if altMaskSpans is not None:
1349 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1350 imageScaler.scaleMaskedImage(exposure.maskedImage)
1351 # Note that the variance plane here is used to store weights, not
1352 # the actual variance
1353 exposure.variance.array[:, :] = 0.0
1354 # Set the weight of unmasked pixels to 1.
1355 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.0
1356 # Set the image value of masked pixels to zero.
1357 # This eliminates needing the mask plane when stacking images in
1358 # ``newModelFromResidual``
1359 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.0
1360 subExposures[visit] = exposure
1361 return subExposures
1363 def selectCoaddPsf(self, templateCoadd, warpRefList):
1364 """Compute the PSF of the coadd from the exposures with the best
1365 seeing.
1367 Parameters
1368 ----------
1369 templateCoadd : `lsst.afw.image.ExposureF`
1370 The initial coadd exposure before accounting for DCR.
1371 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1372 The data references to the input warped exposures.
1374 Returns
1375 -------
1376 psf : `lsst.meas.algorithms.CoaddPsf`
1377 The average PSF of the input exposures with the best seeing.
1378 """
1379 sigma2fwhm = 2.0 * np.sqrt(2.0 * np.log(2.0))
1380 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry
1381 # per ccd and per visit. If there are multiple ccds, it will have that
1382 # many times more elements than ``warpExpRef``.
1383 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1384 templatePsf = templateCoadd.getPsf()
1385 # Just need a rough estimate; average positions are fine
1386 templateAvgPos = templatePsf.getAveragePosition()
1387 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius() * sigma2fwhm
1388 psfSizes = np.zeros(len(ccds))
1389 ccdVisits = np.array(ccds["visit"])
1390 for warpExpRef in warpRefList:
1391 psf = warpExpRef.get(component="psf")
1392 visit = warpExpRef.dataId["visit"]
1393 psfAvgPos = psf.getAveragePosition()
1394 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius() * sigma2fwhm
1395 psfSizes[ccdVisits == visit] = psfSize
1396 # Note that the input PSFs include DCR, which should be absent from the
1397 # DcrCoadd. The selected PSFs are those that have a FWHM less than or
1398 # equal to the smaller of the mean or median FWHM of the input
1399 # exposures.
1400 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1401 goodPsfs = psfSizes <= sizeThreshold
1402 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(), self.config.coaddPsf.makeControl())
1403 return psf