Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py : 76%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23from math import ceil
24import numpy as np
25from scipy import ndimage
26import lsst.geom as geom
27import lsst.afw.image as afwImage
28import lsst.afw.table as afwTable
29import lsst.coadd.utils as coaddUtils
30from lsst.daf.butler import DeferredDatasetHandle
31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32import lsst.meas.algorithms as measAlg
33from lsst.meas.base import SingleFrameMeasurementTask
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.utils as utils
37from .assembleCoadd import (AssembleCoaddTask,
38 CompareWarpAssembleCoaddConfig,
39 CompareWarpAssembleCoaddTask)
40from .coaddBase import makeSkyInfo
41from .measurePsf import MeasurePsfTask
43__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
46class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("tract", "patch", "band", "skymap"),
48 defaultTemplates={"inputCoaddName": "deep",
49 "outputCoaddName": "dcr",
50 "warpType": "direct",
51 "warpTypeSuffix": "",
52 "fakesType": ""}):
53 inputWarps = pipeBase.connectionTypes.Input(
54 doc=("Input list of warps to be assembled i.e. stacked."
55 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56 name="{inputCoaddName}Coadd_{warpType}Warp",
57 storageClass="ExposureF",
58 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59 deferLoad=True,
60 multiple=True
61 )
62 skyMap = pipeBase.connectionTypes.Input(
63 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64 name="{inputCoaddName}Coadd_skyMap",
65 storageClass="SkyMap",
66 dimensions=("skymap", ),
67 )
68 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70 " BRIGHT_OBJECT."),
71 name="brightObjectMask",
72 storageClass="ObjectMaskCatalog",
73 dimensions=("tract", "patch", "skymap", "band"),
74 )
75 templateExposure = pipeBase.connectionTypes.Input(
76 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78 storageClass="ExposureF",
79 dimensions=("tract", "patch", "skymap", "band"),
80 )
81 dcrCoadds = pipeBase.connectionTypes.Output(
82 doc="Output coadded exposure, produced by stacking input warps",
83 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84 storageClass="ExposureF",
85 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
86 multiple=True,
87 )
88 dcrNImages = pipeBase.connectionTypes.Output(
89 doc="Output image of number of input images per pixel",
90 name="{outputCoaddName}Coadd_nImage",
91 storageClass="ImageU",
92 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
93 multiple=True,
94 )
96 def __init__(self, *, config=None):
97 super().__init__(config=config)
98 if not config.doWrite:
99 self.outputs.remove("dcrCoadds")
102class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103 pipelineConnections=DcrAssembleCoaddConnections):
104 dcrNumSubfilters = pexConfig.Field(
105 dtype=int,
106 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107 default=3,
108 )
109 maxNumIter = pexConfig.Field(
110 dtype=int,
111 optional=True,
112 doc="Maximum number of iterations of forward modeling.",
113 default=None,
114 )
115 minNumIter = pexConfig.Field(
116 dtype=int,
117 optional=True,
118 doc="Minimum number of iterations of forward modeling.",
119 default=None,
120 )
121 convergenceThreshold = pexConfig.Field(
122 dtype=float,
123 doc="Target relative change in convergence between iterations of forward modeling.",
124 default=0.001,
125 )
126 useConvergence = pexConfig.Field(
127 dtype=bool,
128 doc="Use convergence test as a forward modeling end condition?"
129 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130 default=True,
131 )
132 baseGain = pexConfig.Field(
133 dtype=float,
134 optional=True,
135 doc="Relative weight to give the new solution vs. the last solution when updating the model."
136 "A value of 1.0 gives equal weight to both solutions."
137 "Small values imply slower convergence of the solution, but can "
138 "help prevent overshooting and failures in the fit."
139 "If ``baseGain`` is None, a conservative gain "
140 "will be calculated from the number of subfilters. ",
141 default=None,
142 )
143 useProgressiveGain = pexConfig.Field(
144 dtype=bool,
145 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146 "When calculating the next gain, we use up to 5 previous gains and convergence values."
147 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148 default=True,
149 )
150 doAirmassWeight = pexConfig.Field(
151 dtype=bool,
152 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153 default=False,
154 )
155 modelWeightsWidth = pexConfig.Field(
156 dtype=float,
157 doc="Width of the region around detected sources to include in the DcrModel.",
158 default=3,
159 )
160 useModelWeights = pexConfig.Field(
161 dtype=bool,
162 doc="Width of the region around detected sources to include in the DcrModel.",
163 default=True,
164 )
165 splitSubfilters = pexConfig.Field(
166 dtype=bool,
167 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168 "Instead of at the midpoint",
169 default=True,
170 )
171 splitThreshold = pexConfig.Field(
172 dtype=float,
173 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174 "Set to 0 to always split the subfilters.",
175 default=0.1,
176 )
177 regularizeModelIterations = pexConfig.Field(
178 dtype=float,
179 doc="Maximum relative change of the model allowed between iterations."
180 "Set to zero to disable.",
181 default=2.,
182 )
183 regularizeModelFrequency = pexConfig.Field(
184 dtype=float,
185 doc="Maximum relative change of the model allowed between subfilters."
186 "Set to zero to disable.",
187 default=4.,
188 )
189 convergenceMaskPlanes = pexConfig.ListField(
190 dtype=str,
191 default=["DETECTED"],
192 doc="Mask planes to use to calculate convergence."
193 )
194 regularizationWidth = pexConfig.Field(
195 dtype=int,
196 default=2,
197 doc="Minimum radius of a region to include in regularization, in pixels."
198 )
199 imageInterpOrder = pexConfig.Field(
200 dtype=int,
201 doc="The order of the spline interpolation used to shift the image plane.",
202 default=3,
203 )
204 accelerateModel = pexConfig.Field(
205 dtype=float,
206 doc="Factor to amplify the differences between model planes by to speed convergence.",
207 default=3,
208 )
209 doCalculatePsf = pexConfig.Field(
210 dtype=bool,
211 doc="Set to detect stars and recalculate the PSF from the final coadd."
212 "Otherwise the PSF is estimated from a selection of the best input exposures",
213 default=False,
214 )
215 detectPsfSources = pexConfig.ConfigurableField(
216 target=measAlg.SourceDetectionTask,
217 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218 )
219 measurePsfSources = pexConfig.ConfigurableField(
220 target=SingleFrameMeasurementTask,
221 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222 )
223 measurePsf = pexConfig.ConfigurableField(
224 target=MeasurePsfTask,
225 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226 )
227 effectiveWavelength = pexConfig.Field(
228 doc="Effective wavelength of the filter, in nm."
229 "Required if transmission curves aren't used."
230 "Support for using transmission curves is to be added in DM-13668.",
231 dtype=float,
232 )
233 bandwidth = pexConfig.Field(
234 doc="Bandwidth of the physical filter, in nm."
235 "Required if transmission curves aren't used."
236 "Support for using transmission curves is to be added in DM-13668.",
237 dtype=float,
238 )
240 def setDefaults(self):
241 CompareWarpAssembleCoaddConfig.setDefaults(self)
242 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
243 self.doNImage = True
244 self.assembleStaticSkyModel.warpType = self.warpType
245 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
246 self.assembleStaticSkyModel.doNImage = False
247 self.assembleStaticSkyModel.doWrite = False
248 self.detectPsfSources.returnOriginalFootprints = False
249 self.detectPsfSources.thresholdPolarity = "positive"
250 # Only use bright sources for PSF measurement
251 self.detectPsfSources.thresholdValue = 50
252 self.detectPsfSources.nSigmaToGrow = 2
253 # A valid star for PSF measurement should at least fill 5x5 pixels
254 self.detectPsfSources.minPixels = 25
255 # Use the variance plane to calculate signal to noise
256 self.detectPsfSources.thresholdType = "pixel_stdev"
257 # The signal to noise limit is good enough, while the flux limit is set
258 # in dimensionless units and may not be appropriate for all data sets.
259 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
262class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
263 """Assemble DCR coadded images from a set of warps.
265 Attributes
266 ----------
267 bufferSize : `int`
268 The number of pixels to grow each subregion by to allow for DCR.
270 Notes
271 -----
272 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
273 Warps (also called coadded temporary exposures), including the effects of
274 Differential Chromatic Refraction (DCR).
275 For full details of the mathematics and algorithm, please see
276 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
278 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
279 each subfilter used in the iterative calculation.
280 It begins by dividing the bandpass-defining filter into N equal bandwidth
281 "subfilters", and divides the flux in each pixel from an initial coadd
282 equally into each as a "dcrModel". Because the airmass and parallactic
283 angle of each individual exposure is known, we can calculate the shift
284 relative to the center of the band in each subfilter due to DCR. For each
285 exposure we apply this shift as a linear transformation to the dcrModels
286 and stack the results to produce a DCR-matched exposure. The matched
287 exposures are subtracted from the input exposures to produce a set of
288 residual images, and these residuals are reverse shifted for each
289 exposures' subfilters and stacked. The shifted and stacked residuals are
290 added to the dcrModels to produce a new estimate of the flux in each pixel
291 within each subfilter. The dcrModels are solved for iteratively, which
292 continues until the solution from a new iteration improves by less than
293 a set percentage, or a maximum number of iterations is reached.
294 Two forms of regularization are employed to reduce unphysical results.
295 First, the new solution is averaged with the solution from the previous
296 iteration, which mitigates oscillating solutions where the model
297 overshoots with alternating very high and low values.
298 Second, a common degeneracy when the data have a limited range of airmass or
299 parallactic angle values is for one subfilter to be fit with very low or
300 negative values, while another subfilter is fit with very high values. This
301 typically appears in the form of holes next to sources in one subfilter,
302 and corresponding extended wings in another. Because each subfilter has
303 a narrow bandwidth we assume that physical sources that are above the noise
304 level will not vary in flux by more than a factor of `frequencyClampFactor`
305 between subfilters, and pixels that have flux deviations larger than that
306 factor will have the excess flux distributed evenly among all subfilters.
307 If `splitSubfilters` is set, then each subfilter will be further sub-
308 divided during the forward modeling step (only). This approximates using
309 a higher number of subfilters that may be necessary for high airmass
310 observations, but does not increase the number of free parameters in the
311 fit. This is needed when there are high airmass observations which would
312 otherwise have significant DCR even within a subfilter. Because calculating
313 the shifted images takes most of the time, splitting the subfilters is
314 turned off by way of the `splitThreshold` option for low-airmass
315 observations that do not suffer from DCR within a subfilter.
316 """
318 ConfigClass = DcrAssembleCoaddConfig
319 _DefaultName = "dcrAssembleCoadd"
321 def __init__(self, *args, **kwargs):
322 super().__init__(*args, **kwargs)
323 if self.config.doCalculatePsf: 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true
324 self.schema = afwTable.SourceTable.makeMinimalSchema()
325 self.makeSubtask("detectPsfSources", schema=self.schema)
326 self.makeSubtask("measurePsfSources", schema=self.schema)
327 self.makeSubtask("measurePsf", schema=self.schema)
329 @utils.inheritDoc(pipeBase.PipelineTask)
330 def runQuantum(self, butlerQC, inputRefs, outputRefs):
331 # Docstring to be formatted with info from PipelineTask.runQuantum
332 """
333 Notes
334 -----
335 Assemble a coadd from a set of Warps.
337 PipelineTask (Gen3) entry point to Coadd a set of Warps.
338 Analogous to `runDataRef`, it prepares all the data products to be
339 passed to `run`, and processes the results before returning a struct
340 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
341 Therefore, its inputs are accessed subregion by subregion
342 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
343 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
344 correspond to an update in `runDataRef` while both entry points
345 are used.
346 """
347 inputData = butlerQC.get(inputRefs)
349 # Construct skyInfo expected by run
350 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
351 skyMap = inputData["skyMap"]
352 outputDataId = butlerQC.quantum.dataId
354 inputData['skyInfo'] = makeSkyInfo(skyMap,
355 tractId=outputDataId['tract'],
356 patchId=outputDataId['patch'])
358 # Construct list of input Deferred Datasets
359 # These quack a bit like like Gen2 DataRefs
360 warpRefList = inputData['inputWarps']
361 # Perform same middle steps as `runDataRef` does
362 inputs = self.prepareInputs(warpRefList)
363 self.log.info("Found %d %s", len(inputs.tempExpRefList),
364 self.getTempExpDatasetName(self.warpType))
365 if len(inputs.tempExpRefList) == 0:
366 self.log.warn("No coadd temporary exposures found")
367 return
369 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
370 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
371 inputs.weightList, supplementaryData=supplementaryData)
373 inputData.setdefault('brightObjectMask', None)
374 for subfilter in range(self.config.dcrNumSubfilters):
375 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
376 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
377 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
379 if self.config.doWrite:
380 butlerQC.put(retStruct, outputRefs)
381 return retStruct
383 @pipeBase.timeMethod
384 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
385 """Assemble a coadd from a set of warps.
387 Coadd a set of Warps. Compute weights to be applied to each Warp and
388 find scalings to match the photometric zeropoint to a reference Warp.
389 Assemble the Warps using run method.
390 Forward model chromatic effects across multiple subfilters,
391 and subtract from the input Warps to build sets of residuals.
392 Use the residuals to construct a new ``DcrModel`` for each subfilter,
393 and iterate until the model converges.
394 Interpolate over NaNs and optionally write the coadd to disk.
395 Return the coadded exposure.
397 Parameters
398 ----------
399 dataRef : `lsst.daf.persistence.ButlerDataRef`
400 Data reference defining the patch for coaddition and the
401 reference Warp
402 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
403 List of data references to warps. Data to be coadded will be
404 selected from this list based on overlap with the patch defined by
405 the data reference.
407 Returns
408 -------
409 results : `lsst.pipe.base.Struct`
410 The Struct contains the following fields:
412 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
413 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
414 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
415 - ``dcrNImages``: `list` of exposure count images for each subfilter
416 """
417 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
418 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
420 skyInfo = self.getSkyInfo(dataRef)
421 if warpRefList is None:
422 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
423 if len(calExpRefList) == 0:
424 self.log.warn("No exposures to coadd")
425 return
426 self.log.info("Coadding %d exposures", len(calExpRefList))
428 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
430 inputData = self.prepareInputs(warpRefList)
431 self.log.info("Found %d %s", len(inputData.tempExpRefList),
432 self.getTempExpDatasetName(self.warpType))
433 if len(inputData.tempExpRefList) == 0:
434 self.log.warn("No coadd temporary exposures found")
435 return
437 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
439 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
440 inputData.weightList, supplementaryData=supplementaryData)
441 if results is None:
442 self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
443 skyInfo.patchInfo.getIndex())
444 return
446 if self.config.doCalculatePsf:
447 self.measureCoaddPsf(results.coaddExposure)
448 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
449 for subfilter in range(self.config.dcrNumSubfilters):
450 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
451 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
452 self.processResults(results.dcrCoadds[subfilter],
453 brightObjectMasks=brightObjects, dataId=dataRef.dataId)
454 if self.config.doWrite:
455 self.log.info("Persisting dcrCoadd")
456 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
457 numSubfilters=self.config.dcrNumSubfilters)
458 if self.config.doNImage and results.dcrNImages is not None:
459 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
460 numSubfilters=self.config.dcrNumSubfilters)
462 return results
464 @utils.inheritDoc(AssembleCoaddTask)
465 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
466 """Load the previously-generated template coadd.
468 This can be removed entirely once we no longer support the Gen 2 butler.
470 Returns
471 -------
472 templateCoadd : `lsst.pipe.base.Struct`
473 Result struct with components:
475 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
476 """
477 templateCoadd = butlerQC.get(inputRefs.templateExposure)
479 return pipeBase.Struct(templateCoadd=templateCoadd)
481 def measureCoaddPsf(self, coaddExposure):
482 """Detect sources on the coadd exposure and measure the final PSF.
484 Parameters
485 ----------
486 coaddExposure : `lsst.afw.image.Exposure`
487 The final coadded exposure.
488 """
489 table = afwTable.SourceTable.make(self.schema)
490 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
491 coaddSources = detResults.sources
492 self.measurePsfSources.run(
493 measCat=coaddSources,
494 exposure=coaddExposure
495 )
496 # Measure the PSF on the stacked subfilter coadds if possible.
497 # We should already have a decent estimate of the coadd PSF, however,
498 # so in case of any errors simply log them as a warning and use the
499 # default PSF.
500 try:
501 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
502 except Exception as e:
503 self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
504 else:
505 coaddExposure.setPsf(psfResults.psf)
507 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
508 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
510 Sets the property ``bufferSize``.
512 Parameters
513 ----------
514 templateCoadd : `lsst.afw.image.ExposureF`
515 The initial coadd exposure before accounting for DCR.
516 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
517 `lsst.daf.persistence.ButlerDataRef`
518 The data references to the input warped exposures.
519 weightList : `list` of `float`
520 The weight to give each input exposure in the coadd
521 Will be modified in place if ``doAirmassWeight`` is set.
523 Returns
524 -------
525 dcrModels : `lsst.pipe.tasks.DcrModel`
526 Best fit model of the true sky after correcting chromatic effects.
528 Raises
529 ------
530 NotImplementedError
531 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
532 """
533 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
534 filterInfo = templateCoadd.getFilter()
535 tempExpName = self.getTempExpDatasetName(self.warpType)
536 dcrShifts = []
537 airmassDict = {}
538 angleDict = {}
539 psfSizeDict = {}
540 for visitNum, warpExpRef in enumerate(warpRefList):
541 if isinstance(warpExpRef, DeferredDatasetHandle):
542 # Gen 3 API
543 visitInfo = warpExpRef.get(component="visitInfo")
544 psf = warpExpRef.get(component="psf")
545 else:
546 # Gen 2 API. Delete this when Gen 2 retired
547 visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
548 psf = warpExpRef.get(tempExpName).getPsf()
549 visit = warpExpRef.dataId["visit"]
550 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
551 airmass = visitInfo.getBoresightAirmass()
552 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
553 airmassDict[visit] = airmass
554 angleDict[visit] = parallacticAngle
555 psfSizeDict[visit] = psfSize
556 if self.config.doAirmassWeight: 556 ↛ 557line 556 didn't jump to line 557, because the condition on line 556 was never true
557 weightList[visitNum] *= airmass
558 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
559 self.config.effectiveWavelength,
560 self.config.bandwidth,
561 self.config.dcrNumSubfilters))))
562 self.log.info("Selected airmasses:\n%s", airmassDict)
563 self.log.info("Selected parallactic angles:\n%s", angleDict)
564 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
565 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
566 try:
567 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
568 except Exception as e:
569 self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e)
570 else:
571 psf = templateCoadd.getPsf()
572 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
573 self.config.dcrNumSubfilters,
574 effectiveWavelength=self.config.effectiveWavelength,
575 bandwidth=self.config.bandwidth,
576 filterInfo=filterInfo,
577 psf=psf)
578 return dcrModels
580 @pipeBase.timeMethod
581 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
582 supplementaryData=None):
583 """Assemble the coadd.
585 Requires additional inputs Struct ``supplementaryData`` to contain a
586 ``templateCoadd`` that serves as the model of the static sky.
588 Find artifacts and apply them to the warps' masks creating a list of
589 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
590 Then pass these alternative masks to the base class's assemble method.
592 Divide the ``templateCoadd`` evenly between each subfilter of a
593 ``DcrModel`` as the starting best estimate of the true wavelength-
594 dependent sky. Forward model the ``DcrModel`` using the known
595 chromatic effects in each subfilter and calculate a convergence metric
596 based on how well the modeled template matches the input warps. If
597 the convergence has not yet reached the desired threshold, then shift
598 and stack the residual images to build a new ``DcrModel``. Apply
599 conditioning to prevent oscillating solutions between iterations or
600 between subfilters.
602 Once the ``DcrModel`` reaches convergence or the maximum number of
603 iterations has been reached, fill the metadata for each subfilter
604 image and make them proper ``coaddExposure``s.
606 Parameters
607 ----------
608 skyInfo : `lsst.pipe.base.Struct`
609 Patch geometry information, from getSkyInfo
610 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
611 `lsst.daf.persistence.ButlerDataRef`
612 The data references to the input warped exposures.
613 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
614 The image scalars correct for the zero point of the exposures.
615 weightList : `list` of `float`
616 The weight to give each input exposure in the coadd
617 supplementaryData : `lsst.pipe.base.Struct`
618 Result struct returned by ``makeSupplementaryData`` with components:
620 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
622 Returns
623 -------
624 result : `lsst.pipe.base.Struct`
625 Result struct with components:
627 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
628 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
629 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
630 - ``dcrNImages``: `list` of exposure count images for each subfilter
631 """
632 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
633 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
634 templateCoadd = supplementaryData.templateCoadd
635 baseMask = templateCoadd.mask.clone()
636 # The variance plane is for each subfilter
637 # and should be proportionately lower than the full-band image
638 baseVariance = templateCoadd.variance.clone()
639 baseVariance /= self.config.dcrNumSubfilters
640 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
641 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
642 templateCoadd.setMask(baseMask)
643 badMaskPlanes = self.config.badMaskPlanes[:]
644 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
645 # This is because pixels in observations that are significantly affect by DCR
646 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
647 # but those are necessary to constrain the DCR model.
648 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
650 stats = self.prepareStats(mask=badPixelMask)
651 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
652 if self.config.doNImage: 652 ↛ 662line 652 didn't jump to line 662, because the condition on line 652 was never false
653 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
654 spanSetMaskList, stats.ctrl)
655 nImage = afwImage.ImageU(skyInfo.bbox)
656 # Note that this nImage will be a factor of dcrNumSubfilters higher than
657 # the nImage returned by assembleCoadd for most pixels. This is because each
658 # subfilter may have a different nImage, and fractional values are not allowed.
659 for dcrNImage in dcrNImages:
660 nImage += dcrNImage
661 else:
662 dcrNImages = None
664 subregionSize = geom.Extent2I(*self.config.subregionSize)
665 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
666 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
667 subIter = 0
668 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
669 modelIter = 0
670 subIter += 1
671 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
672 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
673 dcrBBox = geom.Box2I(subBBox)
674 dcrBBox.grow(self.bufferSize)
675 dcrBBox.clip(dcrModels.bbox)
676 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
677 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
678 imageScalerList, spanSetMaskList)
679 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
680 warpRefList, weightList, stats.ctrl)
681 self.log.info("Initial convergence : %s", convergenceMetric)
682 convergenceList = [convergenceMetric]
683 gainList = []
684 convergenceCheck = 1.
685 refImage = templateCoadd.image
686 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
687 gain = self.calculateGain(convergenceList, gainList)
688 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
689 stats.ctrl, convergenceMetric, gain,
690 modelWeights, refImage, dcrWeights)
691 if self.config.useConvergence: 691 ↛ 708line 691 didn't jump to line 708, because the condition on line 691 was never false
692 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
693 warpRefList, weightList, stats.ctrl)
694 if convergenceMetric == 0: 694 ↛ 695line 694 didn't jump to line 695, because the condition on line 694 was never true
695 self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
696 "most likely due to there being no valid data in the region.",
697 skyInfo.patchInfo.getIndex(), subIter)
698 break
699 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
700 if (convergenceCheck < 0) & (modelIter > minNumIter): 700 ↛ 701line 700 didn't jump to line 701, because the condition on line 700 was never true
701 self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
702 "iterations or desired convergence improvement of %s."
703 " Divergence: %s",
704 skyInfo.patchInfo.getIndex(), subIter,
705 self.config.convergenceThreshold, convergenceCheck)
706 break
707 convergenceList.append(convergenceMetric)
708 if modelIter > maxNumIter: 708 ↛ 709line 708 didn't jump to line 709, because the condition on line 708 was never true
709 if self.config.useConvergence:
710 self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
711 "before reaching desired convergence improvement of %s."
712 " Final convergence improvement: %s",
713 skyInfo.patchInfo.getIndex(), subIter,
714 self.config.convergenceThreshold, convergenceCheck)
715 break
717 if self.config.useConvergence: 717 ↛ 720line 717 didn't jump to line 720, because the condition on line 717 was never false
718 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
719 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
720 modelIter += 1
721 else:
722 if self.config.useConvergence: 722 ↛ 727line 722 didn't jump to line 727, because the condition on line 722 was never false
723 self.log.info("Coadd patch %s subregion %s finished with "
724 "convergence metric %s after %s iterations",
725 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
726 else:
727 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
728 skyInfo.patchInfo.getIndex(), subIter, modelIter)
729 if self.config.useConvergence and convergenceMetric > 0: 729 ↛ 668line 729 didn't jump to line 668, because the condition on line 729 was never false
730 self.log.info("Final convergence improvement was %.4f%% overall",
731 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
733 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
734 calibration=self.scaleZeroPoint.getPhotoCalib(),
735 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
736 mask=baseMask,
737 variance=baseVariance)
738 coaddExposure = self.stackCoadd(dcrCoadds)
739 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
740 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
742 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
743 """Calculate the number of exposures contributing to each subfilter.
745 Parameters
746 ----------
747 dcrModels : `lsst.pipe.tasks.DcrModel`
748 Best fit model of the true sky after correcting chromatic effects.
749 bbox : `lsst.geom.box.Box2I`
750 Bounding box of the patch to coadd.
751 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
752 `lsst.daf.persistence.ButlerDataRef`
753 The data references to the input warped exposures.
754 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
755 Each element of the `dict` contains the new mask plane name
756 (e.g. "CLIPPED and/or "NO_DATA") as the key,
757 and the list of SpanSets to apply to the mask.
758 statsCtrl : `lsst.afw.math.StatisticsControl`
759 Statistics control object for coadd
761 Returns
762 -------
763 dcrNImages : `list` of `lsst.afw.image.ImageU`
764 List of exposure count images for each subfilter
765 dcrWeights : `list` of `lsst.afw.image.ImageF`
766 Per-pixel weights for each subfilter.
767 Equal to 1/(number of unmasked images contributing to each pixel).
768 """
769 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
770 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
771 tempExpName = self.getTempExpDatasetName(self.warpType)
772 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
773 if isinstance(warpExpRef, DeferredDatasetHandle):
774 # Gen 3 API
775 exposure = warpExpRef.get(parameters={'bbox': bbox})
776 else:
777 # Gen 2 API. Delete this when Gen 2 retired
778 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
779 visitInfo = exposure.getInfo().getVisitInfo()
780 wcs = exposure.getInfo().getWcs()
781 mask = exposure.mask
782 if altMaskSpans is not None: 782 ↛ 784line 782 didn't jump to line 784, because the condition on line 782 was never false
783 self.applyAltMaskPlanes(mask, altMaskSpans)
784 weightImage = np.zeros_like(exposure.image.array)
785 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
786 # The weights must be shifted in exactly the same way as the residuals,
787 # because they will be used as the denominator in the weighted average of residuals.
788 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
789 dcrModels.effectiveWavelength, dcrModels.bandwidth)
790 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
791 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
792 dcrWeight.array += shiftedWeights
793 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
794 weightsThreshold = 1.
795 goodPix = dcrWeights[0].array > weightsThreshold
796 for weights in dcrWeights[1:]:
797 goodPix = (weights.array > weightsThreshold) & goodPix
798 for subfilter in range(self.config.dcrNumSubfilters):
799 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
800 dcrWeights[subfilter].array[~goodPix] = 0.
801 dcrNImages[subfilter].array[~goodPix] = 0
802 return (dcrNImages, dcrWeights)
804 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
805 statsCtrl, convergenceMetric,
806 gain, modelWeights, refImage, dcrWeights):
807 """Assemble the DCR coadd for a sub-region.
809 Build a DCR-matched template for each input exposure, then shift the
810 residuals according to the DCR in each subfilter.
811 Stack the shifted residuals and apply them as a correction to the
812 solution from the previous iteration.
813 Restrict the new model solutions from varying by more than a factor of
814 `modelClampFactor` from the last solution, and additionally restrict the
815 individual subfilter models from varying by more than a factor of
816 `frequencyClampFactor` from their average.
817 Finally, mitigate potentially oscillating solutions by averaging the new
818 solution with the solution from the previous iteration, weighted by
819 their convergence metric.
821 Parameters
822 ----------
823 dcrModels : `lsst.pipe.tasks.DcrModel`
824 Best fit model of the true sky after correcting chromatic effects.
825 subExposures : `dict` of `lsst.afw.image.ExposureF`
826 The pre-loaded exposures for the current subregion.
827 bbox : `lsst.geom.box.Box2I`
828 Bounding box of the subregion to coadd.
829 dcrBBox : `lsst.geom.box.Box2I`
830 Sub-region of the coadd which includes a buffer to allow for DCR.
831 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
832 `lsst.daf.persistence.ButlerDataRef`
833 The data references to the input warped exposures.
834 statsCtrl : `lsst.afw.math.StatisticsControl`
835 Statistics control object for coadd
836 convergenceMetric : `float`
837 Quality of fit metric for the matched templates of the input images.
838 gain : `float`, optional
839 Relative weight to give the new solution when updating the model.
840 modelWeights : `numpy.ndarray` or `float`
841 A 2D array of weight values that tapers smoothly to zero away from detected sources.
842 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
843 refImage : `lsst.afw.image.Image`
844 A reference image used to supply the default pixel values.
845 dcrWeights : `list` of `lsst.afw.image.Image`
846 Per-pixel weights for each subfilter.
847 Equal to 1/(number of unmasked images contributing to each pixel).
848 """
849 residualGeneratorList = []
851 for warpExpRef in warpRefList:
852 visit = warpExpRef.dataId["visit"]
853 exposure = subExposures[visit]
854 visitInfo = exposure.getInfo().getVisitInfo()
855 wcs = exposure.getInfo().getWcs()
856 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
857 order=self.config.imageInterpOrder,
858 splitSubfilters=self.config.splitSubfilters,
859 splitThreshold=self.config.splitThreshold,
860 amplifyModel=self.config.accelerateModel)
861 residual = exposure.image.array - templateImage.array
862 # Note that the variance plane here is used to store weights, not the actual variance
863 residual *= exposure.variance.array
864 # The residuals are stored as a list of generators.
865 # This allows the residual for a given subfilter and exposure to be created
866 # on the fly, instead of needing to store them all in memory.
867 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
868 dcrModels.effectiveWavelength,
869 dcrModels.bandwidth))
871 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
872 gain=gain,
873 modelWeights=modelWeights,
874 refImage=refImage,
875 dcrWeights=dcrWeights)
876 dcrModels.assign(dcrSubModelOut, bbox)
878 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
879 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
881 Parameters
882 ----------
883 residual : `numpy.ndarray`
884 The residual masked image for one exposure,
885 after subtracting the matched template
886 visitInfo : `lsst.afw.image.VisitInfo`
887 Metadata for the exposure.
888 wcs : `lsst.afw.geom.SkyWcs`
889 Coordinate system definition (wcs) for the exposure.
890 filterInfo : `lsst.afw.image.Filter`
891 The filter definition, set in the current instruments' obs package.
892 Note: this object will be changed in DM-21333.
894 Yields
895 ------
896 residualImage : `numpy.ndarray`
897 The residual image for the next subfilter, shifted for DCR.
898 """
899 # Pre-calculate the spline-filtered residual image, so that step can be
900 # skipped in the shift calculation in `applyDcr`.
901 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
902 # Note that `splitSubfilters` is always turned off in the reverse direction.
903 # This option introduces additional blurring if applied to the residuals.
904 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
905 splitSubfilters=False)
906 for dcr in dcrShift:
907 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
908 doPrefilter=False, order=self.config.imageInterpOrder)
910 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
911 gain, modelWeights, refImage, dcrWeights):
912 """Calculate a new DcrModel from a set of image residuals.
914 Parameters
915 ----------
916 dcrModels : `lsst.pipe.tasks.DcrModel`
917 Current model of the true sky after correcting chromatic effects.
918 residualGeneratorList : `generator` of `numpy.ndarray`
919 The residual image for the next subfilter, shifted for DCR.
920 dcrBBox : `lsst.geom.box.Box2I`
921 Sub-region of the coadd which includes a buffer to allow for DCR.
922 statsCtrl : `lsst.afw.math.StatisticsControl`
923 Statistics control object for coadd
924 gain : `float`
925 Relative weight to give the new solution when updating the model.
926 modelWeights : `numpy.ndarray` or `float`
927 A 2D array of weight values that tapers smoothly to zero away from detected sources.
928 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
929 refImage : `lsst.afw.image.Image`
930 A reference image used to supply the default pixel values.
931 dcrWeights : `list` of `lsst.afw.image.Image`
932 Per-pixel weights for each subfilter.
933 Equal to 1/(number of unmasked images contributing to each pixel).
935 Returns
936 -------
937 dcrModel : `lsst.pipe.tasks.DcrModel`
938 New model of the true sky after correcting chromatic effects.
939 """
940 newModelImages = []
941 for subfilter, model in enumerate(dcrModels):
942 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
943 residual = np.sum(residualsList, axis=0)
944 residual *= dcrWeights[subfilter][dcrBBox].array
945 # `MaskedImage`s only support in-place addition, so rename for readability
946 newModel = model[dcrBBox].clone()
947 newModel.array += residual
948 # Catch any invalid values
949 badPixels = ~np.isfinite(newModel.array)
950 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
951 if self.config.regularizeModelIterations > 0: 951 ↛ 955line 951 didn't jump to line 955, because the condition on line 951 was never false
952 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
953 self.config.regularizeModelIterations,
954 self.config.regularizationWidth)
955 newModelImages.append(newModel)
956 if self.config.regularizeModelFrequency > 0: 956 ↛ 960line 956 didn't jump to line 960, because the condition on line 956 was never false
957 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
958 self.config.regularizeModelFrequency,
959 self.config.regularizationWidth)
960 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
961 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
962 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
963 dcrModels.bandwidth, dcrModels.psf,
964 dcrModels.mask, dcrModels.variance)
966 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
967 """Calculate a quality of fit metric for the matched templates.
969 Parameters
970 ----------
971 dcrModels : `lsst.pipe.tasks.DcrModel`
972 Best fit model of the true sky after correcting chromatic effects.
973 subExposures : `dict` of `lsst.afw.image.ExposureF`
974 The pre-loaded exposures for the current subregion.
975 bbox : `lsst.geom.box.Box2I`
976 Sub-region to coadd
977 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
978 `lsst.daf.persistence.ButlerDataRef`
979 The data references to the input warped exposures.
980 weightList : `list` of `float`
981 The weight to give each input exposure in the coadd
982 statsCtrl : `lsst.afw.math.StatisticsControl`
983 Statistics control object for coadd
985 Returns
986 -------
987 convergenceMetric : `float`
988 Quality of fit metric for all input exposures, within the sub-region
989 """
990 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
991 nSigma = 3.
992 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
993 bufferSize=self.bufferSize)
994 if np.max(significanceImage) == 0: 994 ↛ 995line 994 didn't jump to line 995, because the condition on line 994 was never true
995 significanceImage += 1.
996 weight = 0
997 metric = 0.
998 metricList = {}
999 for warpExpRef, expWeight in zip(warpRefList, weightList):
1000 visit = warpExpRef.dataId["visit"]
1001 exposure = subExposures[visit][bbox]
1002 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
1003 metric += singleMetric
1004 metricList[visit] = singleMetric
1005 weight += 1.
1006 self.log.info("Individual metrics:\n%s", metricList)
1007 return 1.0 if weight == 0.0 else metric/weight
1009 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
1010 """Calculate a quality of fit metric for a single matched template.
1012 Parameters
1013 ----------
1014 dcrModels : `lsst.pipe.tasks.DcrModel`
1015 Best fit model of the true sky after correcting chromatic effects.
1016 exposure : `lsst.afw.image.ExposureF`
1017 The input warped exposure to evaluate.
1018 significanceImage : `numpy.ndarray`
1019 Array of weights for each pixel corresponding to its significance
1020 for the convergence calculation.
1021 statsCtrl : `lsst.afw.math.StatisticsControl`
1022 Statistics control object for coadd
1024 Returns
1025 -------
1026 convergenceMetric : `float`
1027 Quality of fit metric for one exposure, within the sub-region.
1028 """
1029 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1030 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1031 order=self.config.imageInterpOrder,
1032 splitSubfilters=self.config.splitSubfilters,
1033 splitThreshold=self.config.splitThreshold,
1034 amplifyModel=self.config.accelerateModel)
1035 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1036 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1038 finitePixels = np.isfinite(diffVals)
1039 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1040 convergeMaskPixels = exposure.mask.array & convergeMask > 0
1041 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1042 if np.sum(usePixels) == 0: 1042 ↛ 1043line 1042 didn't jump to line 1043, because the condition on line 1042 was never true
1043 metric = 0.
1044 else:
1045 diffUse = diffVals[usePixels]
1046 refUse = refVals[usePixels]
1047 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1048 return metric
1050 def stackCoadd(self, dcrCoadds):
1051 """Add a list of sub-band coadds together.
1053 Parameters
1054 ----------
1055 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1056 A list of coadd exposures, each exposure containing
1057 the model for one subfilter.
1059 Returns
1060 -------
1061 coaddExposure : `lsst.afw.image.ExposureF`
1062 A single coadd exposure that is the sum of the sub-bands.
1063 """
1064 coaddExposure = dcrCoadds[0].clone()
1065 for coadd in dcrCoadds[1:]:
1066 coaddExposure.maskedImage += coadd.maskedImage
1067 return coaddExposure
1069 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1070 mask=None, variance=None):
1071 """Create a list of coadd exposures from a list of masked images.
1073 Parameters
1074 ----------
1075 dcrModels : `lsst.pipe.tasks.DcrModel`
1076 Best fit model of the true sky after correcting chromatic effects.
1077 skyInfo : `lsst.pipe.base.Struct`
1078 Patch geometry information, from getSkyInfo
1079 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1080 `lsst.daf.persistence.ButlerDataRef`
1081 The data references to the input warped exposures.
1082 weightList : `list` of `float`
1083 The weight to give each input exposure in the coadd
1084 calibration : `lsst.afw.Image.PhotoCalib`, optional
1085 Scale factor to set the photometric calibration of an exposure.
1086 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1087 A record of the observations that are included in the coadd.
1088 mask : `lsst.afw.image.Mask`, optional
1089 Optional mask to override the values in the final coadd.
1090 variance : `lsst.afw.image.Image`, optional
1091 Optional variance plane to override the values in the final coadd.
1093 Returns
1094 -------
1095 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1096 A list of coadd exposures, each exposure containing
1097 the model for one subfilter.
1098 """
1099 dcrCoadds = []
1100 refModel = dcrModels.getReferenceImage()
1101 for model in dcrModels:
1102 if self.config.accelerateModel > 1: 1102 ↛ 1104line 1102 didn't jump to line 1104, because the condition on line 1102 was never false
1103 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1104 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1105 if calibration is not None: 1105 ↛ 1107line 1105 didn't jump to line 1107, because the condition on line 1105 was never false
1106 coaddExposure.setPhotoCalib(calibration)
1107 if coaddInputs is not None: 1107 ↛ 1110line 1107 didn't jump to line 1110, because the condition on line 1107 was never false
1108 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1109 # Set the metadata for the coadd, including PSF and aperture corrections.
1110 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1111 # Overwrite the PSF
1112 coaddExposure.setPsf(dcrModels.psf)
1113 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1114 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1115 maskedImage.image = model
1116 maskedImage.mask = dcrModels.mask
1117 maskedImage.variance = dcrModels.variance
1118 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1119 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1120 if mask is not None: 1120 ↛ 1122line 1120 didn't jump to line 1122, because the condition on line 1120 was never false
1121 coaddExposure.setMask(mask)
1122 if variance is not None: 1122 ↛ 1124line 1122 didn't jump to line 1124, because the condition on line 1122 was never false
1123 coaddExposure.setVariance(variance)
1124 dcrCoadds.append(coaddExposure)
1125 return dcrCoadds
1127 def calculateGain(self, convergenceList, gainList):
1128 """Calculate the gain to use for the current iteration.
1130 After calculating a new DcrModel, each value is averaged with the
1131 value in the corresponding pixel from the previous iteration. This
1132 reduces oscillating solutions that iterative techniques are plagued by,
1133 and speeds convergence. By far the biggest changes to the model
1134 happen in the first couple iterations, so we can also use a more
1135 aggressive gain later when the model is changing slowly.
1137 Parameters
1138 ----------
1139 convergenceList : `list` of `float`
1140 The quality of fit metric from each previous iteration.
1141 gainList : `list` of `float`
1142 The gains used in each previous iteration: appended with the new
1143 gain value.
1144 Gains are numbers between ``self.config.baseGain`` and 1.
1146 Returns
1147 -------
1148 gain : `float`
1149 Relative weight to give the new solution when updating the model.
1150 A value of 1.0 gives equal weight to both solutions.
1152 Raises
1153 ------
1154 ValueError
1155 If ``len(convergenceList) != len(gainList)+1``.
1156 """
1157 nIter = len(convergenceList)
1158 if nIter != len(gainList) + 1:
1159 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1160 % (len(convergenceList), len(gainList)))
1162 if self.config.baseGain is None:
1163 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1164 # The more subfilters being modeled, the lower the gain should be.
1165 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1166 else:
1167 baseGain = self.config.baseGain
1169 if self.config.useProgressiveGain and nIter > 2:
1170 # To calculate the best gain to use, compare the past gains that have been used
1171 # with the resulting convergences to estimate the best gain to use.
1172 # Algorithmically, this is a Kalman filter.
1173 # If forward modeling proceeds perfectly, the convergence metric should
1174 # asymptotically approach a final value.
1175 # We can estimate that value from the measured changes in convergence
1176 # weighted by the gains used in each previous iteration.
1177 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1178 for i in range(nIter - 1)]
1179 # The convergence metric is strictly positive, so if the estimated final convergence is
1180 # less than zero, force it to zero.
1181 estFinalConv = np.array(estFinalConv)
1182 estFinalConv[estFinalConv < 0] = 0
1183 # Because the estimate may slowly change over time, only use the most recent measurements.
1184 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1185 lastGain = gainList[-1]
1186 lastConv = convergenceList[-2]
1187 newConv = convergenceList[-1]
1188 # The predicted convergence is the value we would get if the new model calculated
1189 # in the previous iteration was perfect. Recall that the updated model that is
1190 # actually used is the gain-weighted average of the new and old model,
1191 # so the convergence would be similarly weighted.
1192 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1193 # If the measured and predicted convergence are very close, that indicates
1194 # that our forward model is accurate and we can use a more aggressive gain
1195 # If the measured convergence is significantly worse (or better!) than predicted,
1196 # that indicates that the model is not converging as expected and
1197 # we should use a more conservative gain.
1198 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1199 newGain = 1 - abs(delta)
1200 # Average the gains to prevent oscillating solutions.
1201 newGain = (newGain + lastGain)/2.
1202 gain = max(baseGain, newGain)
1203 else:
1204 gain = baseGain
1205 gainList.append(gain)
1206 return gain
1208 def calculateModelWeights(self, dcrModels, dcrBBox):
1209 """Build an array that smoothly tapers to 0 away from detected sources.
1211 Parameters
1212 ----------
1213 dcrModels : `lsst.pipe.tasks.DcrModel`
1214 Best fit model of the true sky after correcting chromatic effects.
1215 dcrBBox : `lsst.geom.box.Box2I`
1216 Sub-region of the coadd which includes a buffer to allow for DCR.
1218 Returns
1219 -------
1220 weights : `numpy.ndarray` or `float`
1221 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1222 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1224 Raises
1225 ------
1226 ValueError
1227 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1228 """
1229 if not self.config.useModelWeights: 1229 ↛ 1230line 1229 didn't jump to line 1230, because the condition on line 1229 was never true
1230 return 1.0
1231 if self.config.modelWeightsWidth < 0: 1231 ↛ 1232line 1231 didn't jump to line 1232, because the condition on line 1231 was never true
1232 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1233 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1234 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1235 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1236 weights[convergeMaskPixels] = 1.
1237 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1238 weights /= np.max(weights)
1239 return weights
1241 def applyModelWeights(self, modelImages, refImage, modelWeights):
1242 """Smoothly replace model pixel values with those from a
1243 reference at locations away from detected sources.
1245 Parameters
1246 ----------
1247 modelImages : `list` of `lsst.afw.image.Image`
1248 The new DCR model images from the current iteration.
1249 The values will be modified in place.
1250 refImage : `lsst.afw.image.MaskedImage`
1251 A reference image used to supply the default pixel values.
1252 modelWeights : `numpy.ndarray` or `float`
1253 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1254 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1255 """
1256 if self.config.useModelWeights: 1256 ↛ exitline 1256 didn't return from function 'applyModelWeights', because the condition on line 1256 was never false
1257 for model in modelImages:
1258 model.array *= modelWeights
1259 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1261 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1262 """Pre-load sub-regions of a list of exposures.
1264 Parameters
1265 ----------
1266 bbox : `lsst.geom.box.Box2I`
1267 Sub-region to coadd
1268 statsCtrl : `lsst.afw.math.StatisticsControl`
1269 Statistics control object for coadd
1270 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1271 `lsst.daf.persistence.ButlerDataRef`
1272 The data references to the input warped exposures.
1273 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1274 The image scalars correct for the zero point of the exposures.
1275 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1276 Each element is dict with keys = mask plane name to add the spans to
1278 Returns
1279 -------
1280 subExposures : `dict`
1281 The `dict` keys are the visit IDs,
1282 and the values are `lsst.afw.image.ExposureF`
1283 The pre-loaded exposures for the current subregion.
1284 The variance plane contains weights, and not the variance
1285 """
1286 tempExpName = self.getTempExpDatasetName(self.warpType)
1287 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1288 subExposures = {}
1289 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1290 if isinstance(warpExpRef, DeferredDatasetHandle):
1291 exposure = warpExpRef.get(parameters={'bbox': bbox})
1292 else:
1293 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1294 visit = warpExpRef.dataId["visit"]
1295 if altMaskSpans is not None: 1295 ↛ 1297line 1295 didn't jump to line 1297, because the condition on line 1295 was never false
1296 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1297 imageScaler.scaleMaskedImage(exposure.maskedImage)
1298 # Note that the variance plane here is used to store weights, not the actual variance
1299 exposure.variance.array[:, :] = 0.
1300 # Set the weight of unmasked pixels to 1.
1301 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1302 # Set the image value of masked pixels to zero.
1303 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1304 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1305 subExposures[visit] = exposure
1306 return subExposures
1308 def selectCoaddPsf(self, templateCoadd, warpRefList):
1309 """Compute the PSF of the coadd from the exposures with the best seeing.
1311 Parameters
1312 ----------
1313 templateCoadd : `lsst.afw.image.ExposureF`
1314 The initial coadd exposure before accounting for DCR.
1315 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1316 `lsst.daf.persistence.ButlerDataRef`
1317 The data references to the input warped exposures.
1319 Returns
1320 -------
1321 psf : `lsst.meas.algorithms.CoaddPsf`
1322 The average PSF of the input exposures with the best seeing.
1323 """
1324 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1325 tempExpName = self.getTempExpDatasetName(self.warpType)
1326 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1327 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1328 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1329 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1330 psfSizes = np.zeros(len(ccds))
1331 ccdVisits = np.array(ccds["visit"])
1332 for warpExpRef in warpRefList:
1333 if isinstance(warpExpRef, DeferredDatasetHandle):
1334 # Gen 3 API
1335 psf = warpExpRef.get(component="psf")
1336 else:
1337 # Gen 2 API. Delete this when Gen 2 retired
1338 psf = warpExpRef.get(tempExpName).getPsf()
1339 visit = warpExpRef.dataId["visit"]
1340 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1341 psfSizes[ccdVisits == visit] = psfSize
1342 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1343 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1344 # of the mean or median FWHM of the input exposures.
1345 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1346 goodPsfs = psfSizes <= sizeThreshold
1347 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1348 self.config.coaddPsf.makeControl())
1349 return psf