Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py: 15%
414 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 13:00 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 13:00 -0700
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23from math import ceil
24import numpy as np
25from scipy import ndimage
26import lsst.geom as geom
27import lsst.afw.image as afwImage
28import lsst.afw.table as afwTable
29import lsst.coadd.utils as coaddUtils
30from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
31import lsst.meas.algorithms as measAlg
32from lsst.meas.base import SingleFrameMeasurementTask
33import lsst.pex.config as pexConfig
34import lsst.pipe.base as pipeBase
35import lsst.utils as utils
36from lsst.utils.timer import timeMethod
37from .assembleCoadd import (AssembleCoaddConnections,
38 AssembleCoaddTask,
39 CompareWarpAssembleCoaddConfig,
40 CompareWarpAssembleCoaddTask)
41from .coaddBase import makeSkyInfo
42from .measurePsf import MeasurePsfTask
44__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
47class DcrAssembleCoaddConnections(AssembleCoaddConnections,
48 dimensions=("tract", "patch", "band", "skymap"),
49 defaultTemplates={"inputWarpName": "deep",
50 "inputCoaddName": "deep",
51 "outputCoaddName": "dcr",
52 "warpType": "direct",
53 "warpTypeSuffix": "",
54 "fakesType": ""}):
55 inputWarps = pipeBase.connectionTypes.Input(
56 doc=("Input list of warps to be assembled i.e. stacked."
57 "Note that this will often be different than the inputCoaddName."
58 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
59 name="{inputWarpName}Coadd_{warpType}Warp",
60 storageClass="ExposureF",
61 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
62 deferLoad=True,
63 multiple=True
64 )
65 templateExposure = pipeBase.connectionTypes.Input(
66 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
67 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
68 storageClass="ExposureF",
69 dimensions=("tract", "patch", "skymap", "band"),
70 )
71 dcrCoadds = pipeBase.connectionTypes.Output(
72 doc="Output coadded exposure, produced by stacking input warps",
73 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
74 storageClass="ExposureF",
75 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
76 multiple=True,
77 )
78 dcrNImages = pipeBase.connectionTypes.Output(
79 doc="Output image of number of input images per pixel",
80 name="{outputCoaddName}Coadd_nImage",
81 storageClass="ImageU",
82 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
83 multiple=True,
84 )
86 def __init__(self, *, config=None):
87 super().__init__(config=config)
88 if not config.doWrite:
89 self.outputs.remove("dcrCoadds")
90 if not config.doNImage:
91 self.outputs.remove("dcrNImages")
92 # Remove outputs inherited from ``AssembleCoaddConnections`` that are not used
93 self.outputs.remove("coaddExposure")
94 self.outputs.remove("nImage")
97class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
98 pipelineConnections=DcrAssembleCoaddConnections):
99 dcrNumSubfilters = pexConfig.Field(
100 dtype=int,
101 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
102 default=3,
103 )
104 maxNumIter = pexConfig.Field(
105 dtype=int,
106 optional=True,
107 doc="Maximum number of iterations of forward modeling.",
108 default=None,
109 )
110 minNumIter = pexConfig.Field(
111 dtype=int,
112 optional=True,
113 doc="Minimum number of iterations of forward modeling.",
114 default=None,
115 )
116 convergenceThreshold = pexConfig.Field(
117 dtype=float,
118 doc="Target relative change in convergence between iterations of forward modeling.",
119 default=0.001,
120 )
121 useConvergence = pexConfig.Field(
122 dtype=bool,
123 doc="Use convergence test as a forward modeling end condition?"
124 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
125 default=True,
126 )
127 baseGain = pexConfig.Field(
128 dtype=float,
129 optional=True,
130 doc="Relative weight to give the new solution vs. the last solution when updating the model."
131 "A value of 1.0 gives equal weight to both solutions."
132 "Small values imply slower convergence of the solution, but can "
133 "help prevent overshooting and failures in the fit."
134 "If ``baseGain`` is None, a conservative gain "
135 "will be calculated from the number of subfilters. ",
136 default=None,
137 )
138 useProgressiveGain = pexConfig.Field(
139 dtype=bool,
140 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
141 "When calculating the next gain, we use up to 5 previous gains and convergence values."
142 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
143 default=True,
144 )
145 doAirmassWeight = pexConfig.Field(
146 dtype=bool,
147 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
148 default=False,
149 )
150 modelWeightsWidth = pexConfig.Field(
151 dtype=float,
152 doc="Width of the region around detected sources to include in the DcrModel.",
153 default=3,
154 )
155 useModelWeights = pexConfig.Field(
156 dtype=bool,
157 doc="Width of the region around detected sources to include in the DcrModel.",
158 default=True,
159 )
160 splitSubfilters = pexConfig.Field(
161 dtype=bool,
162 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
163 "Instead of at the midpoint",
164 default=True,
165 )
166 splitThreshold = pexConfig.Field(
167 dtype=float,
168 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
169 "Set to 0 to always split the subfilters.",
170 default=0.1,
171 )
172 regularizeModelIterations = pexConfig.Field(
173 dtype=float,
174 doc="Maximum relative change of the model allowed between iterations."
175 "Set to zero to disable.",
176 default=2.,
177 )
178 regularizeModelFrequency = pexConfig.Field(
179 dtype=float,
180 doc="Maximum relative change of the model allowed between subfilters."
181 "Set to zero to disable.",
182 default=4.,
183 )
184 convergenceMaskPlanes = pexConfig.ListField(
185 dtype=str,
186 default=["DETECTED"],
187 doc="Mask planes to use to calculate convergence."
188 )
189 regularizationWidth = pexConfig.Field(
190 dtype=int,
191 default=2,
192 doc="Minimum radius of a region to include in regularization, in pixels."
193 )
194 imageInterpOrder = pexConfig.Field(
195 dtype=int,
196 doc="The order of the spline interpolation used to shift the image plane.",
197 default=1,
198 )
199 accelerateModel = pexConfig.Field(
200 dtype=float,
201 doc="Factor to amplify the differences between model planes by to speed convergence.",
202 default=3,
203 )
204 doCalculatePsf = pexConfig.Field(
205 dtype=bool,
206 doc="Set to detect stars and recalculate the PSF from the final coadd."
207 "Otherwise the PSF is estimated from a selection of the best input exposures",
208 default=False,
209 )
210 detectPsfSources = pexConfig.ConfigurableField(
211 target=measAlg.SourceDetectionTask,
212 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
213 )
214 measurePsfSources = pexConfig.ConfigurableField(
215 target=SingleFrameMeasurementTask,
216 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
217 )
218 measurePsf = pexConfig.ConfigurableField(
219 target=MeasurePsfTask,
220 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
221 )
222 effectiveWavelength = pexConfig.Field(
223 doc="Effective wavelength of the filter, in nm."
224 "Required if transmission curves aren't used."
225 "Support for using transmission curves is to be added in DM-13668.",
226 dtype=float,
227 )
228 bandwidth = pexConfig.Field(
229 doc="Bandwidth of the physical filter, in nm."
230 "Required if transmission curves aren't used."
231 "Support for using transmission curves is to be added in DM-13668.",
232 dtype=float,
233 )
235 def setDefaults(self):
236 CompareWarpAssembleCoaddConfig.setDefaults(self)
237 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
238 self.doNImage = True
239 self.assembleStaticSkyModel.warpType = self.warpType
240 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
241 self.assembleStaticSkyModel.doNImage = False
242 self.assembleStaticSkyModel.doWrite = False
243 self.detectPsfSources.returnOriginalFootprints = False
244 self.detectPsfSources.thresholdPolarity = "positive"
245 # Only use bright sources for PSF measurement
246 self.detectPsfSources.thresholdValue = 50
247 self.detectPsfSources.nSigmaToGrow = 2
248 # A valid star for PSF measurement should at least fill 5x5 pixels
249 self.detectPsfSources.minPixels = 25
250 # Use the variance plane to calculate signal to noise
251 self.detectPsfSources.thresholdType = "pixel_stdev"
252 # The signal to noise limit is good enough, while the flux limit is set
253 # in dimensionless units and may not be appropriate for all data sets.
254 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
255 # Ensure psf candidate size is as large as piff psf size.
256 if (self.doCalculatePsf and self.measurePsf.psfDeterminer.name == "piff"
257 and self.psfDeterminer["piff"].kernelSize > self.makePsfCandidates.kernelSize):
258 self.makePsfCandidates.kernelSize = self.psfDeterminer["piff"].kernelSize
261class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
262 """Assemble DCR coadded images from a set of warps.
264 Attributes
265 ----------
266 bufferSize : `int`
267 The number of pixels to grow each subregion by to allow for DCR.
269 Notes
270 -----
271 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
272 Warps (also called coadded temporary exposures), including the effects of
273 Differential Chromatic Refraction (DCR).
274 For full details of the mathematics and algorithm, please see
275 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
277 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
278 each subfilter used in the iterative calculation.
279 It begins by dividing the bandpass-defining filter into N equal bandwidth
280 "subfilters", and divides the flux in each pixel from an initial coadd
281 equally into each as a "dcrModel". Because the airmass and parallactic
282 angle of each individual exposure is known, we can calculate the shift
283 relative to the center of the band in each subfilter due to DCR. For each
284 exposure we apply this shift as a linear transformation to the dcrModels
285 and stack the results to produce a DCR-matched exposure. The matched
286 exposures are subtracted from the input exposures to produce a set of
287 residual images, and these residuals are reverse shifted for each
288 exposures' subfilters and stacked. The shifted and stacked residuals are
289 added to the dcrModels to produce a new estimate of the flux in each pixel
290 within each subfilter. The dcrModels are solved for iteratively, which
291 continues until the solution from a new iteration improves by less than
292 a set percentage, or a maximum number of iterations is reached.
293 Two forms of regularization are employed to reduce unphysical results.
294 First, the new solution is averaged with the solution from the previous
295 iteration, which mitigates oscillating solutions where the model
296 overshoots with alternating very high and low values.
297 Second, a common degeneracy when the data have a limited range of airmass or
298 parallactic angle values is for one subfilter to be fit with very low or
299 negative values, while another subfilter is fit with very high values. This
300 typically appears in the form of holes next to sources in one subfilter,
301 and corresponding extended wings in another. Because each subfilter has
302 a narrow bandwidth we assume that physical sources that are above the noise
303 level will not vary in flux by more than a factor of `frequencyClampFactor`
304 between subfilters, and pixels that have flux deviations larger than that
305 factor will have the excess flux distributed evenly among all subfilters.
306 If `splitSubfilters` is set, then each subfilter will be further sub-
307 divided during the forward modeling step (only). This approximates using
308 a higher number of subfilters that may be necessary for high airmass
309 observations, but does not increase the number of free parameters in the
310 fit. This is needed when there are high airmass observations which would
311 otherwise have significant DCR even within a subfilter. Because calculating
312 the shifted images takes most of the time, splitting the subfilters is
313 turned off by way of the `splitThreshold` option for low-airmass
314 observations that do not suffer from DCR within a subfilter.
315 """
317 ConfigClass = DcrAssembleCoaddConfig
318 _DefaultName = "dcrAssembleCoadd"
320 def __init__(self, *args, **kwargs):
321 super().__init__(*args, **kwargs)
322 if self.config.doCalculatePsf:
323 self.schema = afwTable.SourceTable.makeMinimalSchema()
324 self.makeSubtask("detectPsfSources", schema=self.schema)
325 self.makeSubtask("measurePsfSources", schema=self.schema)
326 self.makeSubtask("measurePsf", schema=self.schema)
328 @utils.inheritDoc(pipeBase.PipelineTask)
329 def runQuantum(self, butlerQC, inputRefs, outputRefs):
330 # Docstring to be formatted with info from PipelineTask.runQuantum
331 """
332 Notes
333 -----
334 Assemble a coadd from a set of Warps.
335 """
336 inputData = butlerQC.get(inputRefs)
338 # Construct skyInfo expected by run
339 # Do not remove skyMap from inputData in case _makeSupplementaryData needs it
340 skyMap = inputData["skyMap"]
341 outputDataId = butlerQC.quantum.dataId
343 inputData['skyInfo'] = makeSkyInfo(skyMap,
344 tractId=outputDataId['tract'],
345 patchId=outputDataId['patch'])
347 # Construct list of input Deferred Datasets
348 warpRefList = inputData['inputWarps']
350 inputs = self.prepareInputs(warpRefList)
351 self.log.info("Found %d %s", len(inputs.tempExpRefList),
352 self.getTempExpDatasetName(self.warpType))
353 if len(inputs.tempExpRefList) == 0:
354 self.log.warning("No coadd temporary exposures found")
355 return
357 supplementaryData = self._makeSupplementaryData(butlerQC, inputRefs, outputRefs)
358 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
359 inputs.weightList, supplementaryData=supplementaryData)
361 inputData.setdefault('brightObjectMask', None)
362 for subfilter in range(self.config.dcrNumSubfilters):
363 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
364 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
365 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
367 if self.config.doWrite:
368 butlerQC.put(retStruct, outputRefs)
369 return retStruct
371 @utils.inheritDoc(AssembleCoaddTask)
372 def _makeSupplementaryData(self, butlerQC, inputRefs, outputRefs):
373 """Load the previously-generated template coadd.
375 Returns
376 -------
377 templateCoadd : `lsst.pipe.base.Struct`
378 Result struct with components:
380 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
381 """
382 templateCoadd = butlerQC.get(inputRefs.templateExposure)
384 return pipeBase.Struct(templateCoadd=templateCoadd)
386 def measureCoaddPsf(self, coaddExposure):
387 """Detect sources on the coadd exposure and measure the final PSF.
389 Parameters
390 ----------
391 coaddExposure : `lsst.afw.image.Exposure`
392 The final coadded exposure.
393 """
394 table = afwTable.SourceTable.make(self.schema)
395 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
396 coaddSources = detResults.sources
397 self.measurePsfSources.run(
398 measCat=coaddSources,
399 exposure=coaddExposure
400 )
401 # Measure the PSF on the stacked subfilter coadds if possible.
402 # We should already have a decent estimate of the coadd PSF, however,
403 # so in case of any errors simply log them as a warning and use the
404 # default PSF.
405 try:
406 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
407 except Exception as e:
408 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e)
409 else:
410 coaddExposure.setPsf(psfResults.psf)
412 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
413 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
415 Sets the property ``bufferSize``.
417 Parameters
418 ----------
419 templateCoadd : `lsst.afw.image.ExposureF`
420 The initial coadd exposure before accounting for DCR.
421 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
422 The data references to the input warped exposures.
423 weightList : `list` of `float`
424 The weight to give each input exposure in the coadd
425 Will be modified in place if ``doAirmassWeight`` is set.
427 Returns
428 -------
429 dcrModels : `lsst.pipe.tasks.DcrModel`
430 Best fit model of the true sky after correcting chromatic effects.
432 Raises
433 ------
434 NotImplementedError
435 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
436 """
437 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
438 filterLabel = templateCoadd.getFilter()
439 dcrShifts = []
440 airmassDict = {}
441 angleDict = {}
442 psfSizeDict = {}
443 for visitNum, warpExpRef in enumerate(warpRefList):
444 visitInfo = warpExpRef.get(component="visitInfo")
445 psf = warpExpRef.get(component="psf")
446 visit = warpExpRef.dataId["visit"]
447 # Just need a rough estimate; average positions are fine
448 psfAvgPos = psf.getAveragePosition()
449 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
450 airmass = visitInfo.getBoresightAirmass()
451 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
452 airmassDict[visit] = airmass
453 angleDict[visit] = parallacticAngle
454 psfSizeDict[visit] = psfSize
455 if self.config.doAirmassWeight:
456 weightList[visitNum] *= airmass
457 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
458 self.config.effectiveWavelength,
459 self.config.bandwidth,
460 self.config.dcrNumSubfilters))))
461 self.log.info("Selected airmasses:\n%s", airmassDict)
462 self.log.info("Selected parallactic angles:\n%s", angleDict)
463 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
464 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
465 try:
466 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
467 except Exception as e:
468 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e)
469 else:
470 psf = templateCoadd.getPsf()
471 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
472 self.config.dcrNumSubfilters,
473 effectiveWavelength=self.config.effectiveWavelength,
474 bandwidth=self.config.bandwidth,
475 wcs=templateCoadd.getWcs(),
476 filterLabel=filterLabel,
477 psf=psf)
478 return dcrModels
480 @timeMethod
481 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
482 supplementaryData=None):
483 """Assemble the coadd.
485 Requires additional inputs Struct ``supplementaryData`` to contain a
486 ``templateCoadd`` that serves as the model of the static sky.
488 Find artifacts and apply them to the warps' masks creating a list of
489 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
490 Then pass these alternative masks to the base class's assemble method.
492 Divide the ``templateCoadd`` evenly between each subfilter of a
493 ``DcrModel`` as the starting best estimate of the true wavelength-
494 dependent sky. Forward model the ``DcrModel`` using the known
495 chromatic effects in each subfilter and calculate a convergence metric
496 based on how well the modeled template matches the input warps. If
497 the convergence has not yet reached the desired threshold, then shift
498 and stack the residual images to build a new ``DcrModel``. Apply
499 conditioning to prevent oscillating solutions between iterations or
500 between subfilters.
502 Once the ``DcrModel`` reaches convergence or the maximum number of
503 iterations has been reached, fill the metadata for each subfilter
504 image and make them proper ``coaddExposure``s.
506 Parameters
507 ----------
508 skyInfo : `lsst.pipe.base.Struct`
509 Patch geometry information, from getSkyInfo
510 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
511 The data references to the input warped exposures.
512 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
513 The image scalars correct for the zero point of the exposures.
514 weightList : `list` of `float`
515 The weight to give each input exposure in the coadd
516 supplementaryData : `lsst.pipe.base.Struct`
517 Result struct returned by ``_makeSupplementaryData`` with components:
519 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
521 Returns
522 -------
523 result : `lsst.pipe.base.Struct`
524 Result struct with components:
526 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
527 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
528 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
529 - ``dcrNImages``: `list` of exposure count images for each subfilter
530 """
531 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
532 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
533 templateCoadd = supplementaryData.templateCoadd
534 baseMask = templateCoadd.mask.clone()
535 # The variance plane is for each subfilter
536 # and should be proportionately lower than the full-band image
537 baseVariance = templateCoadd.variance.clone()
538 baseVariance /= self.config.dcrNumSubfilters
539 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
540 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
541 templateCoadd.setMask(baseMask)
542 badMaskPlanes = self.config.badMaskPlanes[:]
543 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
544 # This is because pixels in observations that are significantly affect by DCR
545 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
546 # but those are necessary to constrain the DCR model.
547 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
549 stats = self.prepareStats(mask=badPixelMask)
550 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
551 if self.config.doNImage:
552 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
553 spanSetMaskList, stats.ctrl)
554 nImage = afwImage.ImageU(skyInfo.bbox)
555 # Note that this nImage will be a factor of dcrNumSubfilters higher than
556 # the nImage returned by assembleCoadd for most pixels. This is because each
557 # subfilter may have a different nImage, and fractional values are not allowed.
558 for dcrNImage in dcrNImages:
559 nImage += dcrNImage
560 else:
561 dcrNImages = None
563 subregionSize = geom.Extent2I(*self.config.subregionSize)
564 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
565 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
566 subIter = 0
567 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
568 modelIter = 0
569 subIter += 1
570 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
571 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
572 dcrBBox = geom.Box2I(subBBox)
573 dcrBBox.grow(self.bufferSize)
574 dcrBBox.clip(dcrModels.bbox)
575 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
576 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
577 imageScalerList, spanSetMaskList)
578 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
579 warpRefList, weightList, stats.ctrl)
580 self.log.info("Initial convergence : %s", convergenceMetric)
581 convergenceList = [convergenceMetric]
582 gainList = []
583 convergenceCheck = 1.
584 refImage = templateCoadd.image
585 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
586 gain = self.calculateGain(convergenceList, gainList)
587 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
588 stats.ctrl, convergenceMetric, gain,
589 modelWeights, refImage, dcrWeights)
590 if self.config.useConvergence:
591 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
592 warpRefList, weightList, stats.ctrl)
593 if convergenceMetric == 0:
594 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
595 "most likely due to there being no valid data in the region.",
596 skyInfo.patchInfo.getIndex(), subIter)
597 break
598 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
599 if (convergenceCheck < 0) & (modelIter > minNumIter):
600 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum "
601 "iterations or desired convergence improvement of %s."
602 " Divergence: %s",
603 skyInfo.patchInfo.getIndex(), subIter,
604 self.config.convergenceThreshold, convergenceCheck)
605 break
606 convergenceList.append(convergenceMetric)
607 if modelIter > maxNumIter:
608 if self.config.useConvergence:
609 self.log.warning("Coadd patch %s subregion %s reached maximum iterations "
610 "before reaching desired convergence improvement of %s."
611 " Final convergence improvement: %s",
612 skyInfo.patchInfo.getIndex(), subIter,
613 self.config.convergenceThreshold, convergenceCheck)
614 break
616 if self.config.useConvergence:
617 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
618 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
619 modelIter += 1
620 else:
621 if self.config.useConvergence:
622 self.log.info("Coadd patch %s subregion %s finished with "
623 "convergence metric %s after %s iterations",
624 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
625 else:
626 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
627 skyInfo.patchInfo.getIndex(), subIter, modelIter)
628 if self.config.useConvergence and convergenceMetric > 0:
629 self.log.info("Final convergence improvement was %.4f%% overall",
630 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
632 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
633 calibration=self.scaleZeroPoint.getPhotoCalib(),
634 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
635 mask=baseMask,
636 variance=baseVariance)
637 coaddExposure = self.stackCoadd(dcrCoadds)
638 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
639 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
641 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
642 """Calculate the number of exposures contributing to each subfilter.
644 Parameters
645 ----------
646 dcrModels : `lsst.pipe.tasks.DcrModel`
647 Best fit model of the true sky after correcting chromatic effects.
648 bbox : `lsst.geom.box.Box2I`
649 Bounding box of the patch to coadd.
650 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
651 The data references to the input warped exposures.
652 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
653 Each element of the `dict` contains the new mask plane name
654 (e.g. "CLIPPED and/or "NO_DATA") as the key,
655 and the list of SpanSets to apply to the mask.
656 statsCtrl : `lsst.afw.math.StatisticsControl`
657 Statistics control object for coadd
659 Returns
660 -------
661 dcrNImages : `list` of `lsst.afw.image.ImageU`
662 List of exposure count images for each subfilter
663 dcrWeights : `list` of `lsst.afw.image.ImageF`
664 Per-pixel weights for each subfilter.
665 Equal to 1/(number of unmasked images contributing to each pixel).
666 """
667 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
668 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
669 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
670 exposure = warpExpRef.get(parameters={'bbox': bbox})
671 visitInfo = exposure.getInfo().getVisitInfo()
672 wcs = exposure.getInfo().getWcs()
673 mask = exposure.mask
674 if altMaskSpans is not None:
675 self.applyAltMaskPlanes(mask, altMaskSpans)
676 weightImage = np.zeros_like(exposure.image.array)
677 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
678 # The weights must be shifted in exactly the same way as the residuals,
679 # because they will be used as the denominator in the weighted average of residuals.
680 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
681 dcrModels.effectiveWavelength, dcrModels.bandwidth)
682 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
683 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
684 dcrWeight.array += shiftedWeights
685 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
686 weightsThreshold = 1.
687 goodPix = dcrWeights[0].array > weightsThreshold
688 for weights in dcrWeights[1:]:
689 goodPix = (weights.array > weightsThreshold) & goodPix
690 for subfilter in range(self.config.dcrNumSubfilters):
691 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
692 dcrWeights[subfilter].array[~goodPix] = 0.
693 dcrNImages[subfilter].array[~goodPix] = 0
694 return (dcrNImages, dcrWeights)
696 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
697 statsCtrl, convergenceMetric,
698 gain, modelWeights, refImage, dcrWeights):
699 """Assemble the DCR coadd for a sub-region.
701 Build a DCR-matched template for each input exposure, then shift the
702 residuals according to the DCR in each subfilter.
703 Stack the shifted residuals and apply them as a correction to the
704 solution from the previous iteration.
705 Restrict the new model solutions from varying by more than a factor of
706 `modelClampFactor` from the last solution, and additionally restrict the
707 individual subfilter models from varying by more than a factor of
708 `frequencyClampFactor` from their average.
709 Finally, mitigate potentially oscillating solutions by averaging the new
710 solution with the solution from the previous iteration, weighted by
711 their convergence metric.
713 Parameters
714 ----------
715 dcrModels : `lsst.pipe.tasks.DcrModel`
716 Best fit model of the true sky after correcting chromatic effects.
717 subExposures : `dict` of `lsst.afw.image.ExposureF`
718 The pre-loaded exposures for the current subregion.
719 bbox : `lsst.geom.box.Box2I`
720 Bounding box of the subregion to coadd.
721 dcrBBox : `lsst.geom.box.Box2I`
722 Sub-region of the coadd which includes a buffer to allow for DCR.
723 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
724 The data references to the input warped exposures.
725 statsCtrl : `lsst.afw.math.StatisticsControl`
726 Statistics control object for coadd
727 convergenceMetric : `float`
728 Quality of fit metric for the matched templates of the input images.
729 gain : `float`, optional
730 Relative weight to give the new solution when updating the model.
731 modelWeights : `numpy.ndarray` or `float`
732 A 2D array of weight values that tapers smoothly to zero away from detected sources.
733 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
734 refImage : `lsst.afw.image.Image`
735 A reference image used to supply the default pixel values.
736 dcrWeights : `list` of `lsst.afw.image.Image`
737 Per-pixel weights for each subfilter.
738 Equal to 1/(number of unmasked images contributing to each pixel).
739 """
740 residualGeneratorList = []
742 for warpExpRef in warpRefList:
743 visit = warpExpRef.dataId["visit"]
744 exposure = subExposures[visit]
745 visitInfo = exposure.getInfo().getVisitInfo()
746 wcs = exposure.getInfo().getWcs()
747 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
748 bbox=exposure.getBBox(),
749 order=self.config.imageInterpOrder,
750 splitSubfilters=self.config.splitSubfilters,
751 splitThreshold=self.config.splitThreshold,
752 amplifyModel=self.config.accelerateModel)
753 residual = exposure.image.array - templateImage.array
754 # Note that the variance plane here is used to store weights, not the actual variance
755 residual *= exposure.variance.array
756 # The residuals are stored as a list of generators.
757 # This allows the residual for a given subfilter and exposure to be created
758 # on the fly, instead of needing to store them all in memory.
759 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
760 dcrModels.effectiveWavelength,
761 dcrModels.bandwidth))
763 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
764 gain=gain,
765 modelWeights=modelWeights,
766 refImage=refImage,
767 dcrWeights=dcrWeights)
768 dcrModels.assign(dcrSubModelOut, bbox)
770 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
771 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
773 Parameters
774 ----------
775 residual : `numpy.ndarray`
776 The residual masked image for one exposure,
777 after subtracting the matched template
778 visitInfo : `lsst.afw.image.VisitInfo`
779 Metadata for the exposure.
780 wcs : `lsst.afw.geom.SkyWcs`
781 Coordinate system definition (wcs) for the exposure.
783 Yields
784 ------
785 residualImage : `numpy.ndarray`
786 The residual image for the next subfilter, shifted for DCR.
787 """
788 if self.config.imageInterpOrder > 1:
789 # Pre-calculate the spline-filtered residual image, so that step can be
790 # skipped in the shift calculation in `applyDcr`.
791 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
792 else:
793 # No need to prefilter if order=1 (it will also raise an error)
794 filteredResidual = residual
795 # Note that `splitSubfilters` is always turned off in the reverse direction.
796 # This option introduces additional blurring if applied to the residuals.
797 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
798 splitSubfilters=False)
799 for dcr in dcrShift:
800 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
801 doPrefilter=False, order=self.config.imageInterpOrder)
803 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
804 gain, modelWeights, refImage, dcrWeights):
805 """Calculate a new DcrModel from a set of image residuals.
807 Parameters
808 ----------
809 dcrModels : `lsst.pipe.tasks.DcrModel`
810 Current model of the true sky after correcting chromatic effects.
811 residualGeneratorList : `generator` of `numpy.ndarray`
812 The residual image for the next subfilter, shifted for DCR.
813 dcrBBox : `lsst.geom.box.Box2I`
814 Sub-region of the coadd which includes a buffer to allow for DCR.
815 statsCtrl : `lsst.afw.math.StatisticsControl`
816 Statistics control object for coadd
817 gain : `float`
818 Relative weight to give the new solution when updating the model.
819 modelWeights : `numpy.ndarray` or `float`
820 A 2D array of weight values that tapers smoothly to zero away from detected sources.
821 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
822 refImage : `lsst.afw.image.Image`
823 A reference image used to supply the default pixel values.
824 dcrWeights : `list` of `lsst.afw.image.Image`
825 Per-pixel weights for each subfilter.
826 Equal to 1/(number of unmasked images contributing to each pixel).
828 Returns
829 -------
830 dcrModel : `lsst.pipe.tasks.DcrModel`
831 New model of the true sky after correcting chromatic effects.
832 """
833 newModelImages = []
834 for subfilter, model in enumerate(dcrModels):
835 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
836 residual = np.sum(residualsList, axis=0)
837 residual *= dcrWeights[subfilter][dcrBBox].array
838 # `MaskedImage`s only support in-place addition, so rename for readability
839 newModel = model[dcrBBox].clone()
840 newModel.array += residual
841 # Catch any invalid values
842 badPixels = ~np.isfinite(newModel.array)
843 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
844 if self.config.regularizeModelIterations > 0:
845 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
846 self.config.regularizeModelIterations,
847 self.config.regularizationWidth)
848 newModelImages.append(newModel)
849 if self.config.regularizeModelFrequency > 0:
850 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
851 self.config.regularizeModelFrequency,
852 self.config.regularizationWidth)
853 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
854 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
855 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
856 dcrModels.bandwidth, dcrModels.psf,
857 dcrModels.mask, dcrModels.variance)
859 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
860 """Calculate a quality of fit metric for the matched templates.
862 Parameters
863 ----------
864 dcrModels : `lsst.pipe.tasks.DcrModel`
865 Best fit model of the true sky after correcting chromatic effects.
866 subExposures : `dict` of `lsst.afw.image.ExposureF`
867 The pre-loaded exposures for the current subregion.
868 bbox : `lsst.geom.box.Box2I`
869 Sub-region to coadd
870 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
871 The data references to the input warped exposures.
872 weightList : `list` of `float`
873 The weight to give each input exposure in the coadd
874 statsCtrl : `lsst.afw.math.StatisticsControl`
875 Statistics control object for coadd
877 Returns
878 -------
879 convergenceMetric : `float`
880 Quality of fit metric for all input exposures, within the sub-region
881 """
882 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
883 nSigma = 3.
884 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
885 bufferSize=self.bufferSize)
886 if np.max(significanceImage) == 0:
887 significanceImage += 1.
888 weight = 0
889 metric = 0.
890 metricList = {}
891 for warpExpRef, expWeight in zip(warpRefList, weightList):
892 visit = warpExpRef.dataId["visit"]
893 exposure = subExposures[visit][bbox]
894 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
895 metric += singleMetric
896 metricList[visit] = singleMetric
897 weight += 1.
898 self.log.info("Individual metrics:\n%s", metricList)
899 return 1.0 if weight == 0.0 else metric/weight
901 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
902 """Calculate a quality of fit metric for a single matched template.
904 Parameters
905 ----------
906 dcrModels : `lsst.pipe.tasks.DcrModel`
907 Best fit model of the true sky after correcting chromatic effects.
908 exposure : `lsst.afw.image.ExposureF`
909 The input warped exposure to evaluate.
910 significanceImage : `numpy.ndarray`
911 Array of weights for each pixel corresponding to its significance
912 for the convergence calculation.
913 statsCtrl : `lsst.afw.math.StatisticsControl`
914 Statistics control object for coadd
916 Returns
917 -------
918 convergenceMetric : `float`
919 Quality of fit metric for one exposure, within the sub-region.
920 """
921 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
922 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
923 bbox=exposure.getBBox(),
924 order=self.config.imageInterpOrder,
925 splitSubfilters=self.config.splitSubfilters,
926 splitThreshold=self.config.splitThreshold,
927 amplifyModel=self.config.accelerateModel)
928 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
929 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
931 finitePixels = np.isfinite(diffVals)
932 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
933 convergeMaskPixels = exposure.mask.array & convergeMask > 0
934 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
935 if np.sum(usePixels) == 0:
936 metric = 0.
937 else:
938 diffUse = diffVals[usePixels]
939 refUse = refVals[usePixels]
940 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
941 return metric
943 def stackCoadd(self, dcrCoadds):
944 """Add a list of sub-band coadds together.
946 Parameters
947 ----------
948 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
949 A list of coadd exposures, each exposure containing
950 the model for one subfilter.
952 Returns
953 -------
954 coaddExposure : `lsst.afw.image.ExposureF`
955 A single coadd exposure that is the sum of the sub-bands.
956 """
957 coaddExposure = dcrCoadds[0].clone()
958 for coadd in dcrCoadds[1:]:
959 coaddExposure.maskedImage += coadd.maskedImage
960 return coaddExposure
962 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
963 mask=None, variance=None):
964 """Create a list of coadd exposures from a list of masked images.
966 Parameters
967 ----------
968 dcrModels : `lsst.pipe.tasks.DcrModel`
969 Best fit model of the true sky after correcting chromatic effects.
970 skyInfo : `lsst.pipe.base.Struct`
971 Patch geometry information, from getSkyInfo
972 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
973 The data references to the input warped exposures.
974 weightList : `list` of `float`
975 The weight to give each input exposure in the coadd
976 calibration : `lsst.afw.Image.PhotoCalib`, optional
977 Scale factor to set the photometric calibration of an exposure.
978 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
979 A record of the observations that are included in the coadd.
980 mask : `lsst.afw.image.Mask`, optional
981 Optional mask to override the values in the final coadd.
982 variance : `lsst.afw.image.Image`, optional
983 Optional variance plane to override the values in the final coadd.
985 Returns
986 -------
987 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
988 A list of coadd exposures, each exposure containing
989 the model for one subfilter.
990 """
991 dcrCoadds = []
992 refModel = dcrModels.getReferenceImage()
993 for model in dcrModels:
994 if self.config.accelerateModel > 1:
995 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
996 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
997 if calibration is not None:
998 coaddExposure.setPhotoCalib(calibration)
999 if coaddInputs is not None:
1000 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1001 # Set the metadata for the coadd, including PSF and aperture corrections.
1002 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1003 # Overwrite the PSF
1004 coaddExposure.setPsf(dcrModels.psf)
1005 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1006 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1007 maskedImage.image = model
1008 maskedImage.mask = dcrModels.mask
1009 maskedImage.variance = dcrModels.variance
1010 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1011 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1012 if mask is not None:
1013 coaddExposure.setMask(mask)
1014 if variance is not None:
1015 coaddExposure.setVariance(variance)
1016 dcrCoadds.append(coaddExposure)
1017 return dcrCoadds
1019 def calculateGain(self, convergenceList, gainList):
1020 """Calculate the gain to use for the current iteration.
1022 After calculating a new DcrModel, each value is averaged with the
1023 value in the corresponding pixel from the previous iteration. This
1024 reduces oscillating solutions that iterative techniques are plagued by,
1025 and speeds convergence. By far the biggest changes to the model
1026 happen in the first couple iterations, so we can also use a more
1027 aggressive gain later when the model is changing slowly.
1029 Parameters
1030 ----------
1031 convergenceList : `list` of `float`
1032 The quality of fit metric from each previous iteration.
1033 gainList : `list` of `float`
1034 The gains used in each previous iteration: appended with the new
1035 gain value.
1036 Gains are numbers between ``self.config.baseGain`` and 1.
1038 Returns
1039 -------
1040 gain : `float`
1041 Relative weight to give the new solution when updating the model.
1042 A value of 1.0 gives equal weight to both solutions.
1044 Raises
1045 ------
1046 ValueError
1047 If ``len(convergenceList) != len(gainList)+1``.
1048 """
1049 nIter = len(convergenceList)
1050 if nIter != len(gainList) + 1:
1051 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1052 % (len(convergenceList), len(gainList)))
1054 if self.config.baseGain is None:
1055 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1056 # The more subfilters being modeled, the lower the gain should be.
1057 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1058 else:
1059 baseGain = self.config.baseGain
1061 if self.config.useProgressiveGain and nIter > 2:
1062 # To calculate the best gain to use, compare the past gains that have been used
1063 # with the resulting convergences to estimate the best gain to use.
1064 # Algorithmically, this is a Kalman filter.
1065 # If forward modeling proceeds perfectly, the convergence metric should
1066 # asymptotically approach a final value.
1067 # We can estimate that value from the measured changes in convergence
1068 # weighted by the gains used in each previous iteration.
1069 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1070 for i in range(nIter - 1)]
1071 # The convergence metric is strictly positive, so if the estimated final convergence is
1072 # less than zero, force it to zero.
1073 estFinalConv = np.array(estFinalConv)
1074 estFinalConv[estFinalConv < 0] = 0
1075 # Because the estimate may slowly change over time, only use the most recent measurements.
1076 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1077 lastGain = gainList[-1]
1078 lastConv = convergenceList[-2]
1079 newConv = convergenceList[-1]
1080 # The predicted convergence is the value we would get if the new model calculated
1081 # in the previous iteration was perfect. Recall that the updated model that is
1082 # actually used is the gain-weighted average of the new and old model,
1083 # so the convergence would be similarly weighted.
1084 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1085 # If the measured and predicted convergence are very close, that indicates
1086 # that our forward model is accurate and we can use a more aggressive gain
1087 # If the measured convergence is significantly worse (or better!) than predicted,
1088 # that indicates that the model is not converging as expected and
1089 # we should use a more conservative gain.
1090 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1091 newGain = 1 - abs(delta)
1092 # Average the gains to prevent oscillating solutions.
1093 newGain = (newGain + lastGain)/2.
1094 gain = max(baseGain, newGain)
1095 else:
1096 gain = baseGain
1097 gainList.append(gain)
1098 return gain
1100 def calculateModelWeights(self, dcrModels, dcrBBox):
1101 """Build an array that smoothly tapers to 0 away from detected sources.
1103 Parameters
1104 ----------
1105 dcrModels : `lsst.pipe.tasks.DcrModel`
1106 Best fit model of the true sky after correcting chromatic effects.
1107 dcrBBox : `lsst.geom.box.Box2I`
1108 Sub-region of the coadd which includes a buffer to allow for DCR.
1110 Returns
1111 -------
1112 weights : `numpy.ndarray` or `float`
1113 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1114 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1116 Raises
1117 ------
1118 ValueError
1119 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1120 """
1121 if not self.config.useModelWeights:
1122 return 1.0
1123 if self.config.modelWeightsWidth < 0:
1124 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1125 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1126 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1127 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1128 weights[convergeMaskPixels] = 1.
1129 weights = ndimage.gaussian_filter(weights, self.config.modelWeightsWidth)
1130 weights /= np.max(weights)
1131 return weights
1133 def applyModelWeights(self, modelImages, refImage, modelWeights):
1134 """Smoothly replace model pixel values with those from a
1135 reference at locations away from detected sources.
1137 Parameters
1138 ----------
1139 modelImages : `list` of `lsst.afw.image.Image`
1140 The new DCR model images from the current iteration.
1141 The values will be modified in place.
1142 refImage : `lsst.afw.image.MaskedImage`
1143 A reference image used to supply the default pixel values.
1144 modelWeights : `numpy.ndarray` or `float`
1145 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1146 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1147 """
1148 if self.config.useModelWeights:
1149 for model in modelImages:
1150 model.array *= modelWeights
1151 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1153 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1154 """Pre-load sub-regions of a list of exposures.
1156 Parameters
1157 ----------
1158 bbox : `lsst.geom.box.Box2I`
1159 Sub-region to coadd
1160 statsCtrl : `lsst.afw.math.StatisticsControl`
1161 Statistics control object for coadd
1162 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1163 The data references to the input warped exposures.
1164 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1165 The image scalars correct for the zero point of the exposures.
1166 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1167 Each element is dict with keys = mask plane name to add the spans to
1169 Returns
1170 -------
1171 subExposures : `dict`
1172 The `dict` keys are the visit IDs,
1173 and the values are `lsst.afw.image.ExposureF`
1174 The pre-loaded exposures for the current subregion.
1175 The variance plane contains weights, and not the variance
1176 """
1177 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1178 subExposures = {}
1179 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1180 exposure = warpExpRef.get(parameters={'bbox': bbox})
1181 visit = warpExpRef.dataId["visit"]
1182 if altMaskSpans is not None:
1183 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1184 imageScaler.scaleMaskedImage(exposure.maskedImage)
1185 # Note that the variance plane here is used to store weights, not the actual variance
1186 exposure.variance.array[:, :] = 0.
1187 # Set the weight of unmasked pixels to 1.
1188 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1189 # Set the image value of masked pixels to zero.
1190 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1191 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1192 subExposures[visit] = exposure
1193 return subExposures
1195 def selectCoaddPsf(self, templateCoadd, warpRefList):
1196 """Compute the PSF of the coadd from the exposures with the best seeing.
1198 Parameters
1199 ----------
1200 templateCoadd : `lsst.afw.image.ExposureF`
1201 The initial coadd exposure before accounting for DCR.
1202 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle`
1203 The data references to the input warped exposures.
1205 Returns
1206 -------
1207 psf : `lsst.meas.algorithms.CoaddPsf`
1208 The average PSF of the input exposures with the best seeing.
1209 """
1210 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1211 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1212 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1213 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1214 templatePsf = templateCoadd.getPsf()
1215 # Just need a rough estimate; average positions are fine
1216 templateAvgPos = templatePsf.getAveragePosition()
1217 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm
1218 psfSizes = np.zeros(len(ccds))
1219 ccdVisits = np.array(ccds["visit"])
1220 for warpExpRef in warpRefList:
1221 psf = warpExpRef.get(component="psf")
1222 visit = warpExpRef.dataId["visit"]
1223 psfAvgPos = psf.getAveragePosition()
1224 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
1225 psfSizes[ccdVisits == visit] = psfSize
1226 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1227 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1228 # of the mean or median FWHM of the input exposures.
1229 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1230 goodPsfs = psfSizes <= sizeThreshold
1231 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1232 self.config.coaddPsf.makeControl())
1233 return psf