Coverage for python/lsst/pipe/tasks/dcrAssembleCoadd.py : 76%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
23from math import ceil
24import numpy as np
25from scipy import ndimage
26import lsst.geom as geom
27import lsst.afw.image as afwImage
28import lsst.afw.table as afwTable
29import lsst.coadd.utils as coaddUtils
30from lsst.daf.butler import DeferredDatasetHandle
31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32import lsst.meas.algorithms as measAlg
33from lsst.meas.base import SingleFrameMeasurementTask
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.utils as utils
37from .assembleCoadd import (AssembleCoaddTask,
38 CompareWarpAssembleCoaddConfig,
39 CompareWarpAssembleCoaddTask)
40from .coaddBase import makeSkyInfo
41from .measurePsf import MeasurePsfTask
43__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
46class DcrAssembleCoaddConnections(pipeBase.PipelineTaskConnections,
47 dimensions=("tract", "patch", "abstract_filter", "skymap"),
48 defaultTemplates={"inputCoaddName": "deep",
49 "outputCoaddName": "dcr",
50 "warpType": "direct",
51 "warpTypeSuffix": "",
52 "fakesType": ""}):
53 inputWarps = pipeBase.connectionTypes.Input(
54 doc=("Input list of warps to be assembled i.e. stacked."
55 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
56 name="{inputCoaddName}Coadd_{warpType}Warp",
57 storageClass="ExposureF",
58 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
59 deferLoad=True,
60 multiple=True
61 )
62 skyMap = pipeBase.connectionTypes.Input(
63 doc="Input definition of geometry/bbox and projection/wcs for coadded exposures",
64 name="{inputCoaddName}Coadd_skyMap",
65 storageClass="SkyMap",
66 dimensions=("skymap", ),
67 )
68 brightObjectMask = pipeBase.connectionTypes.PrerequisiteInput(
69 doc=("Input Bright Object Mask mask produced with external catalogs to be applied to the mask plane"
70 " BRIGHT_OBJECT."),
71 name="brightObjectMask",
72 storageClass="ObjectMaskCatalog",
73 dimensions=("tract", "patch", "skymap", "abstract_filter"),
74 )
75 templateExposure = pipeBase.connectionTypes.Input(
76 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
77 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
78 storageClass="ExposureF",
79 dimensions=("tract", "patch", "skymap", "abstract_filter"),
80 )
81 dcrCoadds = pipeBase.connectionTypes.Output(
82 doc="Output coadded exposure, produced by stacking input warps",
83 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
84 storageClass="ExposureF",
85 dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
86 multiple=True,
87 )
88 dcrNImages = pipeBase.connectionTypes.Output(
89 doc="Output image of number of input images per pixel",
90 name="{outputCoaddName}Coadd_nImage",
91 storageClass="ImageU",
92 dimensions=("tract", "patch", "skymap", "abstract_filter", "subfilter"),
93 multiple=True,
94 )
96 def __init__(self, *, config=None):
97 super().__init__(config=config)
98 if not config.doWrite:
99 self.outputs.remove("dcrCoadds")
102class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
103 pipelineConnections=DcrAssembleCoaddConnections):
104 dcrNumSubfilters = pexConfig.Field(
105 dtype=int,
106 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
107 default=3,
108 )
109 maxNumIter = pexConfig.Field(
110 dtype=int,
111 optional=True,
112 doc="Maximum number of iterations of forward modeling.",
113 default=None,
114 )
115 minNumIter = pexConfig.Field(
116 dtype=int,
117 optional=True,
118 doc="Minimum number of iterations of forward modeling.",
119 default=None,
120 )
121 convergenceThreshold = pexConfig.Field(
122 dtype=float,
123 doc="Target relative change in convergence between iterations of forward modeling.",
124 default=0.001,
125 )
126 useConvergence = pexConfig.Field(
127 dtype=bool,
128 doc="Use convergence test as a forward modeling end condition?"
129 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
130 default=True,
131 )
132 baseGain = pexConfig.Field(
133 dtype=float,
134 optional=True,
135 doc="Relative weight to give the new solution vs. the last solution when updating the model."
136 "A value of 1.0 gives equal weight to both solutions."
137 "Small values imply slower convergence of the solution, but can "
138 "help prevent overshooting and failures in the fit."
139 "If ``baseGain`` is None, a conservative gain "
140 "will be calculated from the number of subfilters. ",
141 default=None,
142 )
143 useProgressiveGain = pexConfig.Field(
144 dtype=bool,
145 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
146 "When calculating the next gain, we use up to 5 previous gains and convergence values."
147 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
148 default=True,
149 )
150 doAirmassWeight = pexConfig.Field(
151 dtype=bool,
152 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
153 default=False,
154 )
155 modelWeightsWidth = pexConfig.Field(
156 dtype=float,
157 doc="Width of the region around detected sources to include in the DcrModel.",
158 default=3,
159 )
160 useModelWeights = pexConfig.Field(
161 dtype=bool,
162 doc="Width of the region around detected sources to include in the DcrModel.",
163 default=True,
164 )
165 splitSubfilters = pexConfig.Field(
166 dtype=bool,
167 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
168 "Instead of at the midpoint",
169 default=True,
170 )
171 splitThreshold = pexConfig.Field(
172 dtype=float,
173 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
174 "Set to 0 to always split the subfilters.",
175 default=0.1,
176 )
177 regularizeModelIterations = pexConfig.Field(
178 dtype=float,
179 doc="Maximum relative change of the model allowed between iterations."
180 "Set to zero to disable.",
181 default=2.,
182 )
183 regularizeModelFrequency = pexConfig.Field(
184 dtype=float,
185 doc="Maximum relative change of the model allowed between subfilters."
186 "Set to zero to disable.",
187 default=4.,
188 )
189 convergenceMaskPlanes = pexConfig.ListField(
190 dtype=str,
191 default=["DETECTED"],
192 doc="Mask planes to use to calculate convergence."
193 )
194 regularizationWidth = pexConfig.Field(
195 dtype=int,
196 default=2,
197 doc="Minimum radius of a region to include in regularization, in pixels."
198 )
199 imageInterpOrder = pexConfig.Field(
200 dtype=int,
201 doc="The order of the spline interpolation used to shift the image plane.",
202 default=3,
203 )
204 accelerateModel = pexConfig.Field(
205 dtype=float,
206 doc="Factor to amplify the differences between model planes by to speed convergence.",
207 default=3,
208 )
209 doCalculatePsf = pexConfig.Field(
210 dtype=bool,
211 doc="Set to detect stars and recalculate the PSF from the final coadd."
212 "Otherwise the PSF is estimated from a selection of the best input exposures",
213 default=False,
214 )
215 detectPsfSources = pexConfig.ConfigurableField(
216 target=measAlg.SourceDetectionTask,
217 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
218 )
219 measurePsfSources = pexConfig.ConfigurableField(
220 target=SingleFrameMeasurementTask,
221 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
222 )
223 measurePsf = pexConfig.ConfigurableField(
224 target=MeasurePsfTask,
225 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
226 )
228 def setDefaults(self):
229 CompareWarpAssembleCoaddConfig.setDefaults(self)
230 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
231 self.doNImage = True
232 self.assembleStaticSkyModel.warpType = self.warpType
233 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
234 self.assembleStaticSkyModel.doNImage = False
235 self.assembleStaticSkyModel.doWrite = False
236 self.detectPsfSources.returnOriginalFootprints = False
237 self.detectPsfSources.thresholdPolarity = "positive"
238 # Only use bright sources for PSF measurement
239 self.detectPsfSources.thresholdValue = 50
240 self.detectPsfSources.nSigmaToGrow = 2
241 # A valid star for PSF measurement should at least fill 5x5 pixels
242 self.detectPsfSources.minPixels = 25
243 # Use the variance plane to calculate signal to noise
244 self.detectPsfSources.thresholdType = "pixel_stdev"
245 # The signal to noise limit is good enough, while the flux limit is set
246 # in dimensionless units and may not be appropriate for all data sets.
247 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
250class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
251 """Assemble DCR coadded images from a set of warps.
253 Attributes
254 ----------
255 bufferSize : `int`
256 The number of pixels to grow each subregion by to allow for DCR.
258 Notes
259 -----
260 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
261 Warps (also called coadded temporary exposures), including the effects of
262 Differential Chromatic Refraction (DCR).
263 For full details of the mathematics and algorithm, please see
264 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
266 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
267 each subfilter used in the iterative calculation.
268 It begins by dividing the bandpass-defining filter into N equal bandwidth
269 "subfilters", and divides the flux in each pixel from an initial coadd
270 equally into each as a "dcrModel". Because the airmass and parallactic
271 angle of each individual exposure is known, we can calculate the shift
272 relative to the center of the band in each subfilter due to DCR. For each
273 exposure we apply this shift as a linear transformation to the dcrModels
274 and stack the results to produce a DCR-matched exposure. The matched
275 exposures are subtracted from the input exposures to produce a set of
276 residual images, and these residuals are reverse shifted for each
277 exposures' subfilters and stacked. The shifted and stacked residuals are
278 added to the dcrModels to produce a new estimate of the flux in each pixel
279 within each subfilter. The dcrModels are solved for iteratively, which
280 continues until the solution from a new iteration improves by less than
281 a set percentage, or a maximum number of iterations is reached.
282 Two forms of regularization are employed to reduce unphysical results.
283 First, the new solution is averaged with the solution from the previous
284 iteration, which mitigates oscillating solutions where the model
285 overshoots with alternating very high and low values.
286 Second, a common degeneracy when the data have a limited range of airmass or
287 parallactic angle values is for one subfilter to be fit with very low or
288 negative values, while another subfilter is fit with very high values. This
289 typically appears in the form of holes next to sources in one subfilter,
290 and corresponding extended wings in another. Because each subfilter has
291 a narrow bandwidth we assume that physical sources that are above the noise
292 level will not vary in flux by more than a factor of `frequencyClampFactor`
293 between subfilters, and pixels that have flux deviations larger than that
294 factor will have the excess flux distributed evenly among all subfilters.
295 If `splitSubfilters` is set, then each subfilter will be further sub-
296 divided during the forward modeling step (only). This approximates using
297 a higher number of subfilters that may be necessary for high airmass
298 observations, but does not increase the number of free parameters in the
299 fit. This is needed when there are high airmass observations which would
300 otherwise have significant DCR even within a subfilter. Because calculating
301 the shifted images takes most of the time, splitting the subfilters is
302 turned off by way of the `splitThreshold` option for low-airmass
303 observations that do not suffer from DCR within a subfilter.
304 """
306 ConfigClass = DcrAssembleCoaddConfig
307 _DefaultName = "dcrAssembleCoadd"
309 def __init__(self, *args, **kwargs):
310 super().__init__(*args, **kwargs)
311 if self.config.doCalculatePsf: 311 ↛ 312line 311 didn't jump to line 312, because the condition on line 311 was never true
312 self.schema = afwTable.SourceTable.makeMinimalSchema()
313 self.makeSubtask("detectPsfSources", schema=self.schema)
314 self.makeSubtask("measurePsfSources", schema=self.schema)
315 self.makeSubtask("measurePsf", schema=self.schema)
317 @utils.inheritDoc(pipeBase.PipelineTask)
318 def runQuantum(self, butlerQC, inputRefs, outputRefs):
319 # Docstring to be formatted with info from PipelineTask.runQuantum
320 """
321 Notes
322 -----
323 Assemble a coadd from a set of Warps.
325 PipelineTask (Gen3) entry point to Coadd a set of Warps.
326 Analogous to `runDataRef`, it prepares all the data products to be
327 passed to `run`, and processes the results before returning a struct
328 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
329 Therefore, its inputs are accessed subregion by subregion
330 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
331 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
332 correspond to an update in `runDataRef` while both entry points
333 are used.
334 """
335 inputData = butlerQC.get(inputRefs)
337 # Construct skyInfo expected by run
338 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
339 skyMap = inputData["skyMap"]
340 outputDataId = butlerQC.quantum.dataId
342 inputData['skyInfo'] = makeSkyInfo(skyMap,
343 tractId=outputDataId['tract'],
344 patchId=outputDataId['patch'])
346 # Construct list of input Deferred Datasets
347 # These quack a bit like like Gen2 DataRefs
348 warpRefList = inputData['inputWarps']
349 # Perform same middle steps as `runDataRef` does
350 inputs = self.prepareInputs(warpRefList)
351 self.log.info("Found %d %s", len(inputs.tempExpRefList),
352 self.getTempExpDatasetName(self.warpType))
353 if len(inputs.tempExpRefList) == 0:
354 self.log.warn("No coadd temporary exposures found")
355 return
357 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
358 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
359 inputs.weightList, supplementaryData=supplementaryData)
361 inputData.setdefault('brightObjectMask', None)
362 for subfilter in range(self.config.dcrNumSubfilters):
363 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
364 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
365 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
367 if self.config.doWrite:
368 butlerQC.put(retStruct, outputRefs)
369 return retStruct
371 @pipeBase.timeMethod
372 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
373 """Assemble a coadd from a set of warps.
375 Coadd a set of Warps. Compute weights to be applied to each Warp and
376 find scalings to match the photometric zeropoint to a reference Warp.
377 Assemble the Warps using run method.
378 Forward model chromatic effects across multiple subfilters,
379 and subtract from the input Warps to build sets of residuals.
380 Use the residuals to construct a new ``DcrModel`` for each subfilter,
381 and iterate until the model converges.
382 Interpolate over NaNs and optionally write the coadd to disk.
383 Return the coadded exposure.
385 Parameters
386 ----------
387 dataRef : `lsst.daf.persistence.ButlerDataRef`
388 Data reference defining the patch for coaddition and the
389 reference Warp
390 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
391 List of data references to warps. Data to be coadded will be
392 selected from this list based on overlap with the patch defined by
393 the data reference.
395 Returns
396 -------
397 results : `lsst.pipe.base.Struct`
398 The Struct contains the following fields:
400 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
401 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
402 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
403 - ``dcrNImages``: `list` of exposure count images for each subfilter
404 """
405 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
406 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
408 skyInfo = self.getSkyInfo(dataRef)
409 if warpRefList is None:
410 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
411 if len(calExpRefList) == 0:
412 self.log.warn("No exposures to coadd")
413 return
414 self.log.info("Coadding %d exposures", len(calExpRefList))
416 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
418 inputData = self.prepareInputs(warpRefList)
419 self.log.info("Found %d %s", len(inputData.tempExpRefList),
420 self.getTempExpDatasetName(self.warpType))
421 if len(inputData.tempExpRefList) == 0:
422 self.log.warn("No coadd temporary exposures found")
423 return
425 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
427 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
428 inputData.weightList, supplementaryData=supplementaryData)
429 if results is None:
430 self.log.warn("Could not construct DcrModel for patch %s: no data to coadd.",
431 skyInfo.patchInfo.getIndex())
432 return
434 if self.config.doCalculatePsf:
435 self.measureCoaddPsf(results.coaddExposure)
436 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
437 for subfilter in range(self.config.dcrNumSubfilters):
438 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
439 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
440 self.processResults(results.dcrCoadds[subfilter],
441 brightObjectMasks=brightObjects, dataId=dataRef.dataId)
442 if self.config.doWrite:
443 self.log.info("Persisting dcrCoadd")
444 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
445 numSubfilters=self.config.dcrNumSubfilters)
446 if self.config.doNImage and results.dcrNImages is not None:
447 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
448 numSubfilters=self.config.dcrNumSubfilters)
450 return results
452 @utils.inheritDoc(AssembleCoaddTask)
453 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
454 """Load the previously-generated template coadd.
456 This can be removed entirely once we no longer support the Gen 2 butler.
458 Returns
459 -------
460 templateCoadd : `lsst.pipe.base.Struct`
461 Result struct with components:
463 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
464 """
465 templateCoadd = butlerQC.get(inputRefs.templateExposure)
467 return pipeBase.Struct(templateCoadd=templateCoadd)
469 def measureCoaddPsf(self, coaddExposure):
470 """Detect sources on the coadd exposure and measure the final PSF.
472 Parameters
473 ----------
474 coaddExposure : `lsst.afw.image.Exposure`
475 The final coadded exposure.
476 """
477 table = afwTable.SourceTable.make(self.schema)
478 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
479 coaddSources = detResults.sources
480 self.measurePsfSources.run(
481 measCat=coaddSources,
482 exposure=coaddExposure
483 )
484 # Measure the PSF on the stacked subfilter coadds if possible.
485 # We should already have a decent estimate of the coadd PSF, however,
486 # so in case of any errors simply log them as a warning and use the
487 # default PSF.
488 try:
489 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
490 except Exception as e:
491 self.log.warn("Unable to calculate PSF, using default coadd PSF: %s" % e)
492 else:
493 coaddExposure.setPsf(psfResults.psf)
495 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
496 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
498 Sets the property ``bufferSize``.
500 Parameters
501 ----------
502 templateCoadd : `lsst.afw.image.ExposureF`
503 The initial coadd exposure before accounting for DCR.
504 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
505 `lsst.daf.persistence.ButlerDataRef`
506 The data references to the input warped exposures.
507 weightList : `list` of `float`
508 The weight to give each input exposure in the coadd
509 Will be modified in place if ``doAirmassWeight`` is set.
511 Returns
512 -------
513 dcrModels : `lsst.pipe.tasks.DcrModel`
514 Best fit model of the true sky after correcting chromatic effects.
516 Raises
517 ------
518 NotImplementedError
519 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
520 """
521 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
522 filterInfo = templateCoadd.getFilter()
523 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()): 523 ↛ 524line 523 didn't jump to line 524, because the condition on line 523 was never true
524 raise NotImplementedError("No minimum/maximum wavelength information found"
525 " in the filter definition! Please add lambdaMin and lambdaMax"
526 " to the Mapper class in your obs package.")
527 tempExpName = self.getTempExpDatasetName(self.warpType)
528 dcrShifts = []
529 airmassDict = {}
530 angleDict = {}
531 psfSizeDict = {}
532 for visitNum, warpExpRef in enumerate(warpRefList):
533 if isinstance(warpExpRef, DeferredDatasetHandle):
534 # Gen 3 API
535 visitInfo = warpExpRef.get(component="visitInfo")
536 psf = warpExpRef.get(component="psf")
537 else:
538 # Gen 2 API. Delete this when Gen 2 retired
539 visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
540 psf = warpExpRef.get(tempExpName).getPsf()
541 visit = warpExpRef.dataId["visit"]
542 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
543 airmass = visitInfo.getBoresightAirmass()
544 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
545 airmassDict[visit] = airmass
546 angleDict[visit] = parallacticAngle
547 psfSizeDict[visit] = psfSize
548 if self.config.doAirmassWeight: 548 ↛ 549line 548 didn't jump to line 549, because the condition on line 548 was never true
549 weightList[visitNum] *= airmass
550 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
551 filterInfo, self.config.dcrNumSubfilters))))
552 self.log.info("Selected airmasses:\n%s", airmassDict)
553 self.log.info("Selected parallactic angles:\n%s", angleDict)
554 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
555 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
556 try:
557 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
558 except Exception as e:
559 self.log.warn("Unable to calculate restricted PSF, using default coadd PSF: %s" % e)
560 else:
561 psf = templateCoadd.getPsf()
562 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
563 self.config.dcrNumSubfilters,
564 filterInfo=filterInfo,
565 psf=psf)
566 return dcrModels
568 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
569 supplementaryData=None):
570 """Assemble the coadd.
572 Requires additional inputs Struct ``supplementaryData`` to contain a
573 ``templateCoadd`` that serves as the model of the static sky.
575 Find artifacts and apply them to the warps' masks creating a list of
576 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
577 Then pass these alternative masks to the base class's assemble method.
579 Divide the ``templateCoadd`` evenly between each subfilter of a
580 ``DcrModel`` as the starting best estimate of the true wavelength-
581 dependent sky. Forward model the ``DcrModel`` using the known
582 chromatic effects in each subfilter and calculate a convergence metric
583 based on how well the modeled template matches the input warps. If
584 the convergence has not yet reached the desired threshold, then shift
585 and stack the residual images to build a new ``DcrModel``. Apply
586 conditioning to prevent oscillating solutions between iterations or
587 between subfilters.
589 Once the ``DcrModel`` reaches convergence or the maximum number of
590 iterations has been reached, fill the metadata for each subfilter
591 image and make them proper ``coaddExposure``s.
593 Parameters
594 ----------
595 skyInfo : `lsst.pipe.base.Struct`
596 Patch geometry information, from getSkyInfo
597 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
598 `lsst.daf.persistence.ButlerDataRef`
599 The data references to the input warped exposures.
600 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
601 The image scalars correct for the zero point of the exposures.
602 weightList : `list` of `float`
603 The weight to give each input exposure in the coadd
604 supplementaryData : `lsst.pipe.base.Struct`
605 Result struct returned by ``makeSupplementaryData`` with components:
607 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
609 Returns
610 -------
611 result : `lsst.pipe.base.Struct`
612 Result struct with components:
614 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
615 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
616 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
617 - ``dcrNImages``: `list` of exposure count images for each subfilter
618 """
619 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
620 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
621 templateCoadd = supplementaryData.templateCoadd
622 baseMask = templateCoadd.mask.clone()
623 # The variance plane is for each subfilter
624 # and should be proportionately lower than the full-band image
625 baseVariance = templateCoadd.variance.clone()
626 baseVariance /= self.config.dcrNumSubfilters
627 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
628 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
629 templateCoadd.setMask(baseMask)
630 badMaskPlanes = self.config.badMaskPlanes[:]
631 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
632 # This is because pixels in observations that are significantly affect by DCR
633 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
634 # but those are necessary to constrain the DCR model.
635 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
637 stats = self.prepareStats(mask=badPixelMask)
638 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
639 if self.config.doNImage: 639 ↛ 649line 639 didn't jump to line 649, because the condition on line 639 was never false
640 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
641 spanSetMaskList, stats.ctrl)
642 nImage = afwImage.ImageU(skyInfo.bbox)
643 # Note that this nImage will be a factor of dcrNumSubfilters higher than
644 # the nImage returned by assembleCoadd for most pixels. This is because each
645 # subfilter may have a different nImage, and fractional values are not allowed.
646 for dcrNImage in dcrNImages:
647 nImage += dcrNImage
648 else:
649 dcrNImages = None
651 subregionSize = geom.Extent2I(*self.config.subregionSize)
652 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
653 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
654 subIter = 0
655 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
656 modelIter = 0
657 subIter += 1
658 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
659 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
660 dcrBBox = geom.Box2I(subBBox)
661 dcrBBox.grow(self.bufferSize)
662 dcrBBox.clip(dcrModels.bbox)
663 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
664 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
665 imageScalerList, spanSetMaskList)
666 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
667 warpRefList, weightList, stats.ctrl)
668 self.log.info("Initial convergence : %s", convergenceMetric)
669 convergenceList = [convergenceMetric]
670 gainList = []
671 convergenceCheck = 1.
672 refImage = templateCoadd.image
673 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
674 gain = self.calculateGain(convergenceList, gainList)
675 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
676 stats.ctrl, convergenceMetric, gain,
677 modelWeights, refImage, dcrWeights)
678 if self.config.useConvergence: 678 ↛ 695line 678 didn't jump to line 695, because the condition on line 678 was never false
679 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
680 warpRefList, weightList, stats.ctrl)
681 if convergenceMetric == 0: 681 ↛ 682line 681 didn't jump to line 682, because the condition on line 681 was never true
682 self.log.warn("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
683 "most likely due to there being no valid data in the region.",
684 skyInfo.patchInfo.getIndex(), subIter)
685 break
686 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
687 if (convergenceCheck < 0) & (modelIter > minNumIter): 687 ↛ 688line 687 didn't jump to line 688, because the condition on line 687 was never true
688 self.log.warn("Coadd patch %s subregion %s diverged before reaching maximum "
689 "iterations or desired convergence improvement of %s."
690 " Divergence: %s",
691 skyInfo.patchInfo.getIndex(), subIter,
692 self.config.convergenceThreshold, convergenceCheck)
693 break
694 convergenceList.append(convergenceMetric)
695 if modelIter > maxNumIter: 695 ↛ 696line 695 didn't jump to line 696, because the condition on line 695 was never true
696 if self.config.useConvergence:
697 self.log.warn("Coadd patch %s subregion %s reached maximum iterations "
698 "before reaching desired convergence improvement of %s."
699 " Final convergence improvement: %s",
700 skyInfo.patchInfo.getIndex(), subIter,
701 self.config.convergenceThreshold, convergenceCheck)
702 break
704 if self.config.useConvergence: 704 ↛ 707line 704 didn't jump to line 707, because the condition on line 704 was never false
705 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
706 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
707 modelIter += 1
708 else:
709 if self.config.useConvergence: 709 ↛ 714line 709 didn't jump to line 714, because the condition on line 709 was never false
710 self.log.info("Coadd patch %s subregion %s finished with "
711 "convergence metric %s after %s iterations",
712 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
713 else:
714 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
715 skyInfo.patchInfo.getIndex(), subIter, modelIter)
716 if self.config.useConvergence and convergenceMetric > 0: 716 ↛ 655line 716 didn't jump to line 655, because the condition on line 716 was never false
717 self.log.info("Final convergence improvement was %.4f%% overall",
718 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
720 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
721 calibration=self.scaleZeroPoint.getPhotoCalib(),
722 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
723 mask=baseMask,
724 variance=baseVariance)
725 coaddExposure = self.stackCoadd(dcrCoadds)
726 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
727 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
729 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
730 """Calculate the number of exposures contributing to each subfilter.
732 Parameters
733 ----------
734 dcrModels : `lsst.pipe.tasks.DcrModel`
735 Best fit model of the true sky after correcting chromatic effects.
736 bbox : `lsst.geom.box.Box2I`
737 Bounding box of the patch to coadd.
738 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
739 `lsst.daf.persistence.ButlerDataRef`
740 The data references to the input warped exposures.
741 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
742 Each element of the `dict` contains the new mask plane name
743 (e.g. "CLIPPED and/or "NO_DATA") as the key,
744 and the list of SpanSets to apply to the mask.
745 statsCtrl : `lsst.afw.math.StatisticsControl`
746 Statistics control object for coadd
748 Returns
749 -------
750 dcrNImages : `list` of `lsst.afw.image.ImageU`
751 List of exposure count images for each subfilter
752 dcrWeights : `list` of `lsst.afw.image.ImageF`
753 Per-pixel weights for each subfilter.
754 Equal to 1/(number of unmasked images contributing to each pixel).
755 """
756 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
757 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
758 tempExpName = self.getTempExpDatasetName(self.warpType)
759 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
760 if isinstance(warpExpRef, DeferredDatasetHandle):
761 # Gen 3 API
762 exposure = warpExpRef.get(parameters={'bbox': bbox})
763 else:
764 # Gen 2 API. Delete this when Gen 2 retired
765 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
766 visitInfo = exposure.getInfo().getVisitInfo()
767 wcs = exposure.getInfo().getWcs()
768 mask = exposure.mask
769 if altMaskSpans is not None: 769 ↛ 771line 769 didn't jump to line 771, because the condition on line 769 was never false
770 self.applyAltMaskPlanes(mask, altMaskSpans)
771 weightImage = np.zeros_like(exposure.image.array)
772 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
773 # The weights must be shifted in exactly the same way as the residuals,
774 # because they will be used as the denominator in the weighted average of residuals.
775 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
776 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
777 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
778 dcrWeight.array += shiftedWeights
779 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
780 weightsThreshold = 1.
781 goodPix = dcrWeights[0].array > weightsThreshold
782 for weights in dcrWeights[1:]:
783 goodPix = (weights.array > weightsThreshold) & goodPix
784 for subfilter in range(self.config.dcrNumSubfilters):
785 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
786 dcrWeights[subfilter].array[~goodPix] = 0.
787 dcrNImages[subfilter].array[~goodPix] = 0
788 return (dcrNImages, dcrWeights)
790 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
791 statsCtrl, convergenceMetric,
792 gain, modelWeights, refImage, dcrWeights):
793 """Assemble the DCR coadd for a sub-region.
795 Build a DCR-matched template for each input exposure, then shift the
796 residuals according to the DCR in each subfilter.
797 Stack the shifted residuals and apply them as a correction to the
798 solution from the previous iteration.
799 Restrict the new model solutions from varying by more than a factor of
800 `modelClampFactor` from the last solution, and additionally restrict the
801 individual subfilter models from varying by more than a factor of
802 `frequencyClampFactor` from their average.
803 Finally, mitigate potentially oscillating solutions by averaging the new
804 solution with the solution from the previous iteration, weighted by
805 their convergence metric.
807 Parameters
808 ----------
809 dcrModels : `lsst.pipe.tasks.DcrModel`
810 Best fit model of the true sky after correcting chromatic effects.
811 subExposures : `dict` of `lsst.afw.image.ExposureF`
812 The pre-loaded exposures for the current subregion.
813 bbox : `lsst.geom.box.Box2I`
814 Bounding box of the subregion to coadd.
815 dcrBBox : `lsst.geom.box.Box2I`
816 Sub-region of the coadd which includes a buffer to allow for DCR.
817 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
818 `lsst.daf.persistence.ButlerDataRef`
819 The data references to the input warped exposures.
820 statsCtrl : `lsst.afw.math.StatisticsControl`
821 Statistics control object for coadd
822 convergenceMetric : `float`
823 Quality of fit metric for the matched templates of the input images.
824 gain : `float`, optional
825 Relative weight to give the new solution when updating the model.
826 modelWeights : `numpy.ndarray` or `float`
827 A 2D array of weight values that tapers smoothly to zero away from detected sources.
828 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
829 refImage : `lsst.afw.image.Image`
830 A reference image used to supply the default pixel values.
831 dcrWeights : `list` of `lsst.afw.image.Image`
832 Per-pixel weights for each subfilter.
833 Equal to 1/(number of unmasked images contributing to each pixel).
834 """
835 residualGeneratorList = []
837 for warpExpRef in warpRefList:
838 visit = warpExpRef.dataId["visit"]
839 exposure = subExposures[visit]
840 visitInfo = exposure.getInfo().getVisitInfo()
841 wcs = exposure.getInfo().getWcs()
842 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
843 order=self.config.imageInterpOrder,
844 splitSubfilters=self.config.splitSubfilters,
845 splitThreshold=self.config.splitThreshold,
846 amplifyModel=self.config.accelerateModel)
847 residual = exposure.image.array - templateImage.array
848 # Note that the variance plane here is used to store weights, not the actual variance
849 residual *= exposure.variance.array
850 # The residuals are stored as a list of generators.
851 # This allows the residual for a given subfilter and exposure to be created
852 # on the fly, instead of needing to store them all in memory.
853 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
855 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
856 gain=gain,
857 modelWeights=modelWeights,
858 refImage=refImage,
859 dcrWeights=dcrWeights)
860 dcrModels.assign(dcrSubModelOut, bbox)
862 def dcrResiduals(self, residual, visitInfo, wcs, filterInfo):
863 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
865 Parameters
866 ----------
867 residual : `numpy.ndarray`
868 The residual masked image for one exposure,
869 after subtracting the matched template
870 visitInfo : `lsst.afw.image.VisitInfo`
871 Metadata for the exposure.
872 wcs : `lsst.afw.geom.SkyWcs`
873 Coordinate system definition (wcs) for the exposure.
874 filterInfo : `lsst.afw.image.Filter`
875 The filter definition, set in the current instruments' obs package.
876 Required for any calculation of DCR, including making matched templates.
878 Yields
879 ------
880 residualImage : `numpy.ndarray`
881 The residual image for the next subfilter, shifted for DCR.
882 """
883 # Pre-calculate the spline-filtered residual image, so that step can be
884 # skipped in the shift calculation in `applyDcr`.
885 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
886 # Note that `splitSubfilters` is always turned off in the reverse direction.
887 # This option introduces additional blurring if applied to the residuals.
888 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
889 splitSubfilters=False)
890 for dcr in dcrShift:
891 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
892 doPrefilter=False, order=self.config.imageInterpOrder)
894 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
895 gain, modelWeights, refImage, dcrWeights):
896 """Calculate a new DcrModel from a set of image residuals.
898 Parameters
899 ----------
900 dcrModels : `lsst.pipe.tasks.DcrModel`
901 Current model of the true sky after correcting chromatic effects.
902 residualGeneratorList : `generator` of `numpy.ndarray`
903 The residual image for the next subfilter, shifted for DCR.
904 dcrBBox : `lsst.geom.box.Box2I`
905 Sub-region of the coadd which includes a buffer to allow for DCR.
906 statsCtrl : `lsst.afw.math.StatisticsControl`
907 Statistics control object for coadd
908 gain : `float`
909 Relative weight to give the new solution when updating the model.
910 modelWeights : `numpy.ndarray` or `float`
911 A 2D array of weight values that tapers smoothly to zero away from detected sources.
912 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
913 refImage : `lsst.afw.image.Image`
914 A reference image used to supply the default pixel values.
915 dcrWeights : `list` of `lsst.afw.image.Image`
916 Per-pixel weights for each subfilter.
917 Equal to 1/(number of unmasked images contributing to each pixel).
919 Returns
920 -------
921 dcrModel : `lsst.pipe.tasks.DcrModel`
922 New model of the true sky after correcting chromatic effects.
923 """
924 newModelImages = []
925 for subfilter, model in enumerate(dcrModels):
926 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
927 residual = np.sum(residualsList, axis=0)
928 residual *= dcrWeights[subfilter][dcrBBox].array
929 # `MaskedImage`s only support in-place addition, so rename for readability
930 newModel = model[dcrBBox].clone()
931 newModel.array += residual
932 # Catch any invalid values
933 badPixels = ~np.isfinite(newModel.array)
934 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
935 if self.config.regularizeModelIterations > 0: 935 ↛ 939line 935 didn't jump to line 939, because the condition on line 935 was never false
936 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
937 self.config.regularizeModelIterations,
938 self.config.regularizationWidth)
939 newModelImages.append(newModel)
940 if self.config.regularizeModelFrequency > 0: 940 ↛ 944line 940 didn't jump to line 944, because the condition on line 940 was never false
941 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
942 self.config.regularizeModelFrequency,
943 self.config.regularizationWidth)
944 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
945 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
946 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
947 dcrModels.mask, dcrModels.variance)
949 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
950 """Calculate a quality of fit metric for the matched templates.
952 Parameters
953 ----------
954 dcrModels : `lsst.pipe.tasks.DcrModel`
955 Best fit model of the true sky after correcting chromatic effects.
956 subExposures : `dict` of `lsst.afw.image.ExposureF`
957 The pre-loaded exposures for the current subregion.
958 bbox : `lsst.geom.box.Box2I`
959 Sub-region to coadd
960 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
961 `lsst.daf.persistence.ButlerDataRef`
962 The data references to the input warped exposures.
963 weightList : `list` of `float`
964 The weight to give each input exposure in the coadd
965 statsCtrl : `lsst.afw.math.StatisticsControl`
966 Statistics control object for coadd
968 Returns
969 -------
970 convergenceMetric : `float`
971 Quality of fit metric for all input exposures, within the sub-region
972 """
973 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
974 nSigma = 3.
975 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
976 bufferSize=self.bufferSize)
977 if np.max(significanceImage) == 0: 977 ↛ 978line 977 didn't jump to line 978, because the condition on line 977 was never true
978 significanceImage += 1.
979 weight = 0
980 metric = 0.
981 metricList = {}
982 for warpExpRef, expWeight in zip(warpRefList, weightList):
983 visit = warpExpRef.dataId["visit"]
984 exposure = subExposures[visit][bbox]
985 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
986 metric += singleMetric
987 metricList[visit] = singleMetric
988 weight += 1.
989 self.log.info("Individual metrics:\n%s", metricList)
990 return 1.0 if weight == 0.0 else metric/weight
992 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
993 """Calculate a quality of fit metric for a single matched template.
995 Parameters
996 ----------
997 dcrModels : `lsst.pipe.tasks.DcrModel`
998 Best fit model of the true sky after correcting chromatic effects.
999 exposure : `lsst.afw.image.ExposureF`
1000 The input warped exposure to evaluate.
1001 significanceImage : `numpy.ndarray`
1002 Array of weights for each pixel corresponding to its significance
1003 for the convergence calculation.
1004 statsCtrl : `lsst.afw.math.StatisticsControl`
1005 Statistics control object for coadd
1007 Returns
1008 -------
1009 convergenceMetric : `float`
1010 Quality of fit metric for one exposure, within the sub-region.
1011 """
1012 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1013 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1014 order=self.config.imageInterpOrder,
1015 splitSubfilters=self.config.splitSubfilters,
1016 splitThreshold=self.config.splitThreshold,
1017 amplifyModel=self.config.accelerateModel)
1018 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1019 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1021 finitePixels = np.isfinite(diffVals)
1022 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1023 convergeMaskPixels = exposure.mask.array & convergeMask > 0
1024 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1025 if np.sum(usePixels) == 0: 1025 ↛ 1026line 1025 didn't jump to line 1026, because the condition on line 1025 was never true
1026 metric = 0.
1027 else:
1028 diffUse = diffVals[usePixels]
1029 refUse = refVals[usePixels]
1030 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1031 return metric
1033 def stackCoadd(self, dcrCoadds):
1034 """Add a list of sub-band coadds together.
1036 Parameters
1037 ----------
1038 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1039 A list of coadd exposures, each exposure containing
1040 the model for one subfilter.
1042 Returns
1043 -------
1044 coaddExposure : `lsst.afw.image.ExposureF`
1045 A single coadd exposure that is the sum of the sub-bands.
1046 """
1047 coaddExposure = dcrCoadds[0].clone()
1048 for coadd in dcrCoadds[1:]:
1049 coaddExposure.maskedImage += coadd.maskedImage
1050 return coaddExposure
1052 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1053 mask=None, variance=None):
1054 """Create a list of coadd exposures from a list of masked images.
1056 Parameters
1057 ----------
1058 dcrModels : `lsst.pipe.tasks.DcrModel`
1059 Best fit model of the true sky after correcting chromatic effects.
1060 skyInfo : `lsst.pipe.base.Struct`
1061 Patch geometry information, from getSkyInfo
1062 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1063 `lsst.daf.persistence.ButlerDataRef`
1064 The data references to the input warped exposures.
1065 weightList : `list` of `float`
1066 The weight to give each input exposure in the coadd
1067 calibration : `lsst.afw.Image.PhotoCalib`, optional
1068 Scale factor to set the photometric calibration of an exposure.
1069 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1070 A record of the observations that are included in the coadd.
1071 mask : `lsst.afw.image.Mask`, optional
1072 Optional mask to override the values in the final coadd.
1073 variance : `lsst.afw.image.Image`, optional
1074 Optional variance plane to override the values in the final coadd.
1076 Returns
1077 -------
1078 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1079 A list of coadd exposures, each exposure containing
1080 the model for one subfilter.
1081 """
1082 dcrCoadds = []
1083 refModel = dcrModels.getReferenceImage()
1084 for model in dcrModels:
1085 if self.config.accelerateModel > 1: 1085 ↛ 1087line 1085 didn't jump to line 1087, because the condition on line 1085 was never false
1086 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1087 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1088 if calibration is not None: 1088 ↛ 1090line 1088 didn't jump to line 1090, because the condition on line 1088 was never false
1089 coaddExposure.setPhotoCalib(calibration)
1090 if coaddInputs is not None: 1090 ↛ 1093line 1090 didn't jump to line 1093, because the condition on line 1090 was never false
1091 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1092 # Set the metadata for the coadd, including PSF and aperture corrections.
1093 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1094 # Overwrite the PSF
1095 coaddExposure.setPsf(dcrModels.psf)
1096 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1097 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1098 maskedImage.image = model
1099 maskedImage.mask = dcrModels.mask
1100 maskedImage.variance = dcrModels.variance
1101 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1102 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1103 if mask is not None: 1103 ↛ 1105line 1103 didn't jump to line 1105, because the condition on line 1103 was never false
1104 coaddExposure.setMask(mask)
1105 if variance is not None: 1105 ↛ 1107line 1105 didn't jump to line 1107, because the condition on line 1105 was never false
1106 coaddExposure.setVariance(variance)
1107 dcrCoadds.append(coaddExposure)
1108 return dcrCoadds
1110 def calculateGain(self, convergenceList, gainList):
1111 """Calculate the gain to use for the current iteration.
1113 After calculating a new DcrModel, each value is averaged with the
1114 value in the corresponding pixel from the previous iteration. This
1115 reduces oscillating solutions that iterative techniques are plagued by,
1116 and speeds convergence. By far the biggest changes to the model
1117 happen in the first couple iterations, so we can also use a more
1118 aggressive gain later when the model is changing slowly.
1120 Parameters
1121 ----------
1122 convergenceList : `list` of `float`
1123 The quality of fit metric from each previous iteration.
1124 gainList : `list` of `float`
1125 The gains used in each previous iteration: appended with the new
1126 gain value.
1127 Gains are numbers between ``self.config.baseGain`` and 1.
1129 Returns
1130 -------
1131 gain : `float`
1132 Relative weight to give the new solution when updating the model.
1133 A value of 1.0 gives equal weight to both solutions.
1135 Raises
1136 ------
1137 ValueError
1138 If ``len(convergenceList) != len(gainList)+1``.
1139 """
1140 nIter = len(convergenceList)
1141 if nIter != len(gainList) + 1:
1142 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1143 % (len(convergenceList), len(gainList)))
1145 if self.config.baseGain is None:
1146 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1147 # The more subfilters being modeled, the lower the gain should be.
1148 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1149 else:
1150 baseGain = self.config.baseGain
1152 if self.config.useProgressiveGain and nIter > 2:
1153 # To calculate the best gain to use, compare the past gains that have been used
1154 # with the resulting convergences to estimate the best gain to use.
1155 # Algorithmically, this is a Kalman filter.
1156 # If forward modeling proceeds perfectly, the convergence metric should
1157 # asymptotically approach a final value.
1158 # We can estimate that value from the measured changes in convergence
1159 # weighted by the gains used in each previous iteration.
1160 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1161 for i in range(nIter - 1)]
1162 # The convergence metric is strictly positive, so if the estimated final convergence is
1163 # less than zero, force it to zero.
1164 estFinalConv = np.array(estFinalConv)
1165 estFinalConv[estFinalConv < 0] = 0
1166 # Because the estimate may slowly change over time, only use the most recent measurements.
1167 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1168 lastGain = gainList[-1]
1169 lastConv = convergenceList[-2]
1170 newConv = convergenceList[-1]
1171 # The predicted convergence is the value we would get if the new model calculated
1172 # in the previous iteration was perfect. Recall that the updated model that is
1173 # actually used is the gain-weighted average of the new and old model,
1174 # so the convergence would be similarly weighted.
1175 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1176 # If the measured and predicted convergence are very close, that indicates
1177 # that our forward model is accurate and we can use a more aggressive gain
1178 # If the measured convergence is significantly worse (or better!) than predicted,
1179 # that indicates that the model is not converging as expected and
1180 # we should use a more conservative gain.
1181 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1182 newGain = 1 - abs(delta)
1183 # Average the gains to prevent oscillating solutions.
1184 newGain = (newGain + lastGain)/2.
1185 gain = max(baseGain, newGain)
1186 else:
1187 gain = baseGain
1188 gainList.append(gain)
1189 return gain
1191 def calculateModelWeights(self, dcrModels, dcrBBox):
1192 """Build an array that smoothly tapers to 0 away from detected sources.
1194 Parameters
1195 ----------
1196 dcrModels : `lsst.pipe.tasks.DcrModel`
1197 Best fit model of the true sky after correcting chromatic effects.
1198 dcrBBox : `lsst.geom.box.Box2I`
1199 Sub-region of the coadd which includes a buffer to allow for DCR.
1201 Returns
1202 -------
1203 weights : `numpy.ndarray` or `float`
1204 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1205 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1207 Raises
1208 ------
1209 ValueError
1210 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1211 """
1212 if not self.config.useModelWeights: 1212 ↛ 1213line 1212 didn't jump to line 1213, because the condition on line 1212 was never true
1213 return 1.0
1214 if self.config.modelWeightsWidth < 0: 1214 ↛ 1215line 1214 didn't jump to line 1215, because the condition on line 1214 was never true
1215 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1216 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1217 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1218 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1219 weights[convergeMaskPixels] = 1.
1220 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1221 weights /= np.max(weights)
1222 return weights
1224 def applyModelWeights(self, modelImages, refImage, modelWeights):
1225 """Smoothly replace model pixel values with those from a
1226 reference at locations away from detected sources.
1228 Parameters
1229 ----------
1230 modelImages : `list` of `lsst.afw.image.Image`
1231 The new DCR model images from the current iteration.
1232 The values will be modified in place.
1233 refImage : `lsst.afw.image.MaskedImage`
1234 A reference image used to supply the default pixel values.
1235 modelWeights : `numpy.ndarray` or `float`
1236 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1237 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1238 """
1239 if self.config.useModelWeights: 1239 ↛ exitline 1239 didn't return from function 'applyModelWeights', because the condition on line 1239 was never false
1240 for model in modelImages:
1241 model.array *= modelWeights
1242 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1244 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1245 """Pre-load sub-regions of a list of exposures.
1247 Parameters
1248 ----------
1249 bbox : `lsst.geom.box.Box2I`
1250 Sub-region to coadd
1251 statsCtrl : `lsst.afw.math.StatisticsControl`
1252 Statistics control object for coadd
1253 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1254 `lsst.daf.persistence.ButlerDataRef`
1255 The data references to the input warped exposures.
1256 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1257 The image scalars correct for the zero point of the exposures.
1258 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1259 Each element is dict with keys = mask plane name to add the spans to
1261 Returns
1262 -------
1263 subExposures : `dict`
1264 The `dict` keys are the visit IDs,
1265 and the values are `lsst.afw.image.ExposureF`
1266 The pre-loaded exposures for the current subregion.
1267 The variance plane contains weights, and not the variance
1268 """
1269 tempExpName = self.getTempExpDatasetName(self.warpType)
1270 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1271 subExposures = {}
1272 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1273 if isinstance(warpExpRef, DeferredDatasetHandle):
1274 exposure = warpExpRef.get(parameters={'bbox': bbox})
1275 else:
1276 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1277 visit = warpExpRef.dataId["visit"]
1278 if altMaskSpans is not None: 1278 ↛ 1280line 1278 didn't jump to line 1280, because the condition on line 1278 was never false
1279 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1280 imageScaler.scaleMaskedImage(exposure.maskedImage)
1281 # Note that the variance plane here is used to store weights, not the actual variance
1282 exposure.variance.array[:, :] = 0.
1283 # Set the weight of unmasked pixels to 1.
1284 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1285 # Set the image value of masked pixels to zero.
1286 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1287 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1288 subExposures[visit] = exposure
1289 return subExposures
1291 def selectCoaddPsf(self, templateCoadd, warpRefList):
1292 """Compute the PSF of the coadd from the exposures with the best seeing.
1294 Parameters
1295 ----------
1296 templateCoadd : `lsst.afw.image.ExposureF`
1297 The initial coadd exposure before accounting for DCR.
1298 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1299 `lsst.daf.persistence.ButlerDataRef`
1300 The data references to the input warped exposures.
1302 Returns
1303 -------
1304 psf : `lsst.meas.algorithms.CoaddPsf`
1305 The average PSF of the input exposures with the best seeing.
1306 """
1307 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1308 tempExpName = self.getTempExpDatasetName(self.warpType)
1309 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1310 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1311 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1312 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1313 psfSizes = np.zeros(len(ccds))
1314 ccdVisits = np.array(ccds["visit"])
1315 for warpExpRef in warpRefList:
1316 if isinstance(warpExpRef, DeferredDatasetHandle):
1317 # Gen 3 API
1318 psf = warpExpRef.get(component="psf")
1319 else:
1320 # Gen 2 API. Delete this when Gen 2 retired
1321 psf = warpExpRef.get(tempExpName).getPsf()
1322 visit = warpExpRef.dataId["visit"]
1323 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1324 psfSizes[ccdVisits == visit] = psfSize
1325 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1326 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1327 # of the mean or median FWHM of the input exposures.
1328 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1329 goodPsfs = psfSizes <= sizeThreshold
1330 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1331 self.config.coaddPsf.makeControl())
1332 return psf