lsst.pipe.tasks gb4d8b8e895+4b50854106
dcrAssembleCoadd.py
Go to the documentation of this file.
1# This file is part of pipe_tasks.
2#
3# LSST Data Management System
4# This product includes software developed by the
5# LSST Project (http://www.lsst.org/).
6# See COPYRIGHT file at the top of the source tree.
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <https://www.lsstcorp.org/LegalNotices/>.
21#
22
23from math import ceil
24import numpy as np
25from scipy import ndimage
26import lsst.geom as geom
27import lsst.afw.image as afwImage
28import lsst.afw.table as afwTable
29import lsst.coadd.utils as coaddUtils
30from lsst.daf.butler import DeferredDatasetHandle
31from lsst.ip.diffim.dcrModel import applyDcr, calculateDcr, DcrModel
32import lsst.meas.algorithms as measAlg
33from lsst.meas.base import SingleFrameMeasurementTask
34import lsst.pex.config as pexConfig
35import lsst.pipe.base as pipeBase
36import lsst.utils as utils
37from lsst.utils.timer import timeMethod
38from .assembleCoadd import (AssembleCoaddConnections,
39 AssembleCoaddTask,
40 CompareWarpAssembleCoaddConfig,
41 CompareWarpAssembleCoaddTask)
42from .coaddBase import makeSkyInfo
43from .measurePsf import MeasurePsfTask
44
45__all__ = ["DcrAssembleCoaddConnections", "DcrAssembleCoaddTask", "DcrAssembleCoaddConfig"]
46
47
49 dimensions=("tract", "patch", "band", "skymap"),
50 defaultTemplates={"inputWarpName": "deep",
51 "inputCoaddName": "deep",
52 "outputCoaddName": "dcr",
53 "warpType": "direct",
54 "warpTypeSuffix": "",
55 "fakesType": ""}):
56 inputWarps = pipeBase.connectionTypes.Input(
57 doc=("Input list of warps to be assembled i.e. stacked."
58 "Note that this will often be different than the inputCoaddName."
59 "WarpType (e.g. direct, psfMatched) is controlled by the warpType config parameter"),
60 name="{inputWarpName}Coadd_{warpType}Warp",
61 storageClass="ExposureF",
62 dimensions=("tract", "patch", "skymap", "visit", "instrument"),
63 deferLoad=True,
64 multiple=True
65 )
66 templateExposure = pipeBase.connectionTypes.Input(
67 doc="Input coadded exposure, produced by previous call to AssembleCoadd",
68 name="{fakesType}{inputCoaddName}Coadd{warpTypeSuffix}",
69 storageClass="ExposureF",
70 dimensions=("tract", "patch", "skymap", "band"),
71 )
72 dcrCoadds = pipeBase.connectionTypes.Output(
73 doc="Output coadded exposure, produced by stacking input warps",
74 name="{fakesType}{outputCoaddName}Coadd{warpTypeSuffix}",
75 storageClass="ExposureF",
76 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
77 multiple=True,
78 )
79 dcrNImages = pipeBase.connectionTypes.Output(
80 doc="Output image of number of input images per pixel",
81 name="{outputCoaddName}Coadd_nImage",
82 storageClass="ImageU",
83 dimensions=("tract", "patch", "skymap", "band", "subfilter"),
84 multiple=True,
85 )
86
87 def __init__(self, *, config=None):
88 super().__init__(config=config)
89 if not config.doWrite:
90 self.outputs.remove("dcrCoadds")
91 if not config.doNImage:
92 self.outputs.remove("dcrNImages")
93 # Remove outputs inherited from ``AssembleCoaddConnections`` that are not used
94 self.outputs.remove("coaddExposure")
95 self.outputs.remove("nImage")
96
97
98class DcrAssembleCoaddConfig(CompareWarpAssembleCoaddConfig,
99 pipelineConnections=DcrAssembleCoaddConnections):
100 dcrNumSubfilters = pexConfig.Field(
101 dtype=int,
102 doc="Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
103 default=3,
104 )
105 maxNumIter = pexConfig.Field(
106 dtype=int,
107 optional=True,
108 doc="Maximum number of iterations of forward modeling.",
109 default=None,
110 )
111 minNumIter = pexConfig.Field(
112 dtype=int,
113 optional=True,
114 doc="Minimum number of iterations of forward modeling.",
115 default=None,
116 )
117 convergenceThreshold = pexConfig.Field(
118 dtype=float,
119 doc="Target relative change in convergence between iterations of forward modeling.",
120 default=0.001,
121 )
122 useConvergence = pexConfig.Field(
123 dtype=bool,
124 doc="Use convergence test as a forward modeling end condition?"
125 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
126 default=True,
127 )
128 baseGain = pexConfig.Field(
129 dtype=float,
130 optional=True,
131 doc="Relative weight to give the new solution vs. the last solution when updating the model."
132 "A value of 1.0 gives equal weight to both solutions."
133 "Small values imply slower convergence of the solution, but can "
134 "help prevent overshooting and failures in the fit."
135 "If ``baseGain`` is None, a conservative gain "
136 "will be calculated from the number of subfilters. ",
137 default=None,
138 )
139 useProgressiveGain = pexConfig.Field(
140 dtype=bool,
141 doc="Use a gain that slowly increases above ``baseGain`` to accelerate convergence? "
142 "When calculating the next gain, we use up to 5 previous gains and convergence values."
143 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
144 default=True,
145 )
146 doAirmassWeight = pexConfig.Field(
147 dtype=bool,
148 doc="Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
149 default=False,
150 )
151 modelWeightsWidth = pexConfig.Field(
152 dtype=float,
153 doc="Width of the region around detected sources to include in the DcrModel.",
154 default=3,
155 )
156 useModelWeights = pexConfig.Field(
157 dtype=bool,
158 doc="Width of the region around detected sources to include in the DcrModel.",
159 default=True,
160 )
161 splitSubfilters = pexConfig.Field(
162 dtype=bool,
163 doc="Calculate DCR for two evenly-spaced wavelengths in each subfilter."
164 "Instead of at the midpoint",
165 default=True,
166 )
167 splitThreshold = pexConfig.Field(
168 dtype=float,
169 doc="Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels."
170 "Set to 0 to always split the subfilters.",
171 default=0.1,
172 )
173 regularizeModelIterations = pexConfig.Field(
174 dtype=float,
175 doc="Maximum relative change of the model allowed between iterations."
176 "Set to zero to disable.",
177 default=2.,
178 )
179 regularizeModelFrequency = pexConfig.Field(
180 dtype=float,
181 doc="Maximum relative change of the model allowed between subfilters."
182 "Set to zero to disable.",
183 default=4.,
184 )
185 convergenceMaskPlanes = pexConfig.ListField(
186 dtype=str,
187 default=["DETECTED"],
188 doc="Mask planes to use to calculate convergence."
189 )
190 regularizationWidth = pexConfig.Field(
191 dtype=int,
192 default=2,
193 doc="Minimum radius of a region to include in regularization, in pixels."
194 )
195 imageInterpOrder = pexConfig.Field(
196 dtype=int,
197 doc="The order of the spline interpolation used to shift the image plane.",
198 default=3,
199 )
200 accelerateModel = pexConfig.Field(
201 dtype=float,
202 doc="Factor to amplify the differences between model planes by to speed convergence.",
203 default=3,
204 )
205 doCalculatePsf = pexConfig.Field(
206 dtype=bool,
207 doc="Set to detect stars and recalculate the PSF from the final coadd."
208 "Otherwise the PSF is estimated from a selection of the best input exposures",
209 default=False,
210 )
211 detectPsfSources = pexConfig.ConfigurableField(
212 target=measAlg.SourceDetectionTask,
213 doc="Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
214 )
215 measurePsfSources = pexConfig.ConfigurableField(
216 target=SingleFrameMeasurementTask,
217 doc="Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set."
218 )
219 measurePsf = pexConfig.ConfigurableField(
220 target=MeasurePsfTask,
221 doc="Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
222 )
223 effectiveWavelength = pexConfig.Field(
224 doc="Effective wavelength of the filter, in nm."
225 "Required if transmission curves aren't used."
226 "Support for using transmission curves is to be added in DM-13668.",
227 dtype=float,
228 )
229 bandwidth = pexConfig.Field(
230 doc="Bandwidth of the physical filter, in nm."
231 "Required if transmission curves aren't used."
232 "Support for using transmission curves is to be added in DM-13668.",
233 dtype=float,
234 )
235
236 def setDefaults(self):
237 CompareWarpAssembleCoaddConfig.setDefaults(self)
238 self.assembleStaticSkyModel.retarget(CompareWarpAssembleCoaddTask)
239 self.doNImage = True
240 self.assembleStaticSkyModel.warpType = self.warpType
241 # The deepCoadd and nImage files will be overwritten by this Task, so don't write them the first time
242 self.assembleStaticSkyModel.doNImage = False
243 self.assembleStaticSkyModel.doWrite = False
244 self.detectPsfSources.returnOriginalFootprints = False
245 self.detectPsfSources.thresholdPolarity = "positive"
246 # Only use bright sources for PSF measurement
247 self.detectPsfSources.thresholdValue = 50
248 self.detectPsfSources.nSigmaToGrow = 2
249 # A valid star for PSF measurement should at least fill 5x5 pixels
250 self.detectPsfSources.minPixels = 25
251 # Use the variance plane to calculate signal to noise
252 self.detectPsfSources.thresholdType = "pixel_stdev"
253 # The signal to noise limit is good enough, while the flux limit is set
254 # in dimensionless units and may not be appropriate for all data sets.
255 self.measurePsf.starSelector["objectSize"].doFluxLimit = False
256
257
258class DcrAssembleCoaddTask(CompareWarpAssembleCoaddTask):
259 """Assemble DCR coadded images from a set of warps.
260
261 Attributes
262 ----------
263 bufferSize : `int`
264 The number of pixels to grow each subregion by to allow for DCR.
265
266 Notes
267 -----
268 As with AssembleCoaddTask, we want to assemble a coadded image from a set of
269 Warps (also called coadded temporary exposures), including the effects of
270 Differential Chromatic Refraction (DCR).
271 For full details of the mathematics and algorithm, please see
272 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io).
273
274 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for
275 each subfilter used in the iterative calculation.
276 It begins by dividing the bandpass-defining filter into N equal bandwidth
277 "subfilters", and divides the flux in each pixel from an initial coadd
278 equally into each as a "dcrModel". Because the airmass and parallactic
279 angle of each individual exposure is known, we can calculate the shift
280 relative to the center of the band in each subfilter due to DCR. For each
281 exposure we apply this shift as a linear transformation to the dcrModels
282 and stack the results to produce a DCR-matched exposure. The matched
283 exposures are subtracted from the input exposures to produce a set of
284 residual images, and these residuals are reverse shifted for each
285 exposures' subfilters and stacked. The shifted and stacked residuals are
286 added to the dcrModels to produce a new estimate of the flux in each pixel
287 within each subfilter. The dcrModels are solved for iteratively, which
288 continues until the solution from a new iteration improves by less than
289 a set percentage, or a maximum number of iterations is reached.
290 Two forms of regularization are employed to reduce unphysical results.
291 First, the new solution is averaged with the solution from the previous
292 iteration, which mitigates oscillating solutions where the model
293 overshoots with alternating very high and low values.
294 Second, a common degeneracy when the data have a limited range of airmass or
295 parallactic angle values is for one subfilter to be fit with very low or
296 negative values, while another subfilter is fit with very high values. This
297 typically appears in the form of holes next to sources in one subfilter,
298 and corresponding extended wings in another. Because each subfilter has
299 a narrow bandwidth we assume that physical sources that are above the noise
300 level will not vary in flux by more than a factor of `frequencyClampFactor`
301 between subfilters, and pixels that have flux deviations larger than that
302 factor will have the excess flux distributed evenly among all subfilters.
303 If `splitSubfilters` is set, then each subfilter will be further sub-
304 divided during the forward modeling step (only). This approximates using
305 a higher number of subfilters that may be necessary for high airmass
306 observations, but does not increase the number of free parameters in the
307 fit. This is needed when there are high airmass observations which would
308 otherwise have significant DCR even within a subfilter. Because calculating
309 the shifted images takes most of the time, splitting the subfilters is
310 turned off by way of the `splitThreshold` option for low-airmass
311 observations that do not suffer from DCR within a subfilter.
312 """
313
314 ConfigClass = DcrAssembleCoaddConfig
315 _DefaultName = "dcrAssembleCoadd"
316
317 def __init__(self, *args, **kwargs):
318 super().__init__(*args, **kwargs)
319 if self.config.doCalculatePsf:
320 self.schema = afwTable.SourceTable.makeMinimalSchema()
321 self.makeSubtask("detectPsfSources", schema=self.schema)
322 self.makeSubtask("measurePsfSources", schema=self.schema)
323 self.makeSubtask("measurePsf", schema=self.schema)
324
325 @utils.inheritDoc(pipeBase.PipelineTask)
326 def runQuantum(self, butlerQC, inputRefs, outputRefs):
327 # Docstring to be formatted with info from PipelineTask.runQuantum
328 """
329 Notes
330 -----
331 Assemble a coadd from a set of Warps.
332
333 PipelineTask (Gen3) entry point to Coadd a set of Warps.
334 Analogous to `runDataRef`, it prepares all the data products to be
335 passed to `run`, and processes the results before returning a struct
336 of results to be written out. AssembleCoadd cannot fit all Warps in memory.
337 Therefore, its inputs are accessed subregion by subregion
338 by the Gen3 `DeferredDatasetHandle` that is analagous to the Gen2
339 `lsst.daf.persistence.ButlerDataRef`. Any updates to this method should
340 correspond to an update in `runDataRef` while both entry points
341 are used.
342 """
343 inputData = butlerQC.get(inputRefs)
344
345 # Construct skyInfo expected by run
346 # Do not remove skyMap from inputData in case makeSupplementaryDataGen3 needs it
347 skyMap = inputData["skyMap"]
348 outputDataId = butlerQC.quantum.dataId
349
350 inputData['skyInfo'] = makeSkyInfo(skyMap,
351 tractId=outputDataId['tract'],
352 patchId=outputDataId['patch'])
353
354 # Construct list of input Deferred Datasets
355 # These quack a bit like like Gen2 DataRefs
356 warpRefList = inputData['inputWarps']
357 # Perform same middle steps as `runDataRef` does
358 inputs = self.prepareInputs(warpRefList)
359 self.log.info("Found %d %s", len(inputs.tempExpRefList),
360 self.getTempExpDatasetName(self.warpType))
361 if len(inputs.tempExpRefList) == 0:
362 self.log.warning("No coadd temporary exposures found")
363 return
364
365 supplementaryData = self.makeSupplementaryDataGen3(butlerQC, inputRefs, outputRefs)
366 retStruct = self.run(inputData['skyInfo'], inputs.tempExpRefList, inputs.imageScalerList,
367 inputs.weightList, supplementaryData=supplementaryData)
368
369 inputData.setdefault('brightObjectMask', None)
370 for subfilter in range(self.config.dcrNumSubfilters):
371 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
372 retStruct.dcrCoadds[subfilter].setPsf(retStruct.coaddExposure.getPsf())
373 self.processResults(retStruct.dcrCoadds[subfilter], inputData['brightObjectMask'], outputDataId)
374
375 if self.config.doWrite:
376 butlerQC.put(retStruct, outputRefs)
377 return retStruct
378
379 @timeMethod
380 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
381 """Assemble a coadd from a set of warps.
382
383 Coadd a set of Warps. Compute weights to be applied to each Warp and
384 find scalings to match the photometric zeropoint to a reference Warp.
385 Assemble the Warps using run method.
386 Forward model chromatic effects across multiple subfilters,
387 and subtract from the input Warps to build sets of residuals.
388 Use the residuals to construct a new ``DcrModel`` for each subfilter,
389 and iterate until the model converges.
390 Interpolate over NaNs and optionally write the coadd to disk.
391 Return the coadded exposure.
392
393 Parameters
394 ----------
395 dataRef : `lsst.daf.persistence.ButlerDataRef`
396 Data reference defining the patch for coaddition and the
397 reference Warp
398 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef`
399 List of data references to warps. Data to be coadded will be
400 selected from this list based on overlap with the patch defined by
401 the data reference.
402
403 Returns
404 -------
405 results : `lsst.pipe.base.Struct`
406 The Struct contains the following fields:
407
408 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
409 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
410 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
411 - ``dcrNImages``: `list` of exposure count images for each subfilter
412 """
413 if (selectDataList is None and warpRefList is None) or (selectDataList and warpRefList):
414 raise RuntimeError("runDataRef must be supplied either a selectDataList or warpRefList")
415
416 skyInfo = self.getSkyInfo(dataRef)
417 if warpRefList is None:
418 calExpRefList = self.selectExposures(dataRef, skyInfo, selectDataList=selectDataList)
419 if len(calExpRefList) == 0:
420 self.log.warning("No exposures to coadd")
421 return
422 self.log.info("Coadding %d exposures", len(calExpRefList))
423
424 warpRefList = self.getTempExpRefList(dataRef, calExpRefList)
425
426 inputData = self.prepareInputs(warpRefList)
427 self.log.info("Found %d %s", len(inputData.tempExpRefList),
428 self.getTempExpDatasetName(self.warpType))
429 if len(inputData.tempExpRefList) == 0:
430 self.log.warning("No coadd temporary exposures found")
431 return
432
433 supplementaryData = self.makeSupplementaryData(dataRef, warpRefList=inputData.tempExpRefList)
434
435 results = self.run(skyInfo, inputData.tempExpRefList, inputData.imageScalerList,
436 inputData.weightList, supplementaryData=supplementaryData)
437 if results is None:
438 self.log.warning("Could not construct DcrModel for patch %s: no data to coadd.",
439 skyInfo.patchInfo.getIndex())
440 return
441
442 if self.config.doCalculatePsf:
443 self.measureCoaddPsf(results.coaddExposure)
444 brightObjects = self.readBrightObjectMasks(dataRef) if self.config.doMaskBrightObjects else None
445 for subfilter in range(self.config.dcrNumSubfilters):
446 # Use the PSF of the stacked dcrModel, and do not recalculate the PSF for each subfilter
447 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
448 self.processResults(results.dcrCoadds[subfilter],
449 brightObjectMasks=brightObjects, dataId=dataRef.dataId)
450 if self.config.doWrite:
451 self.log.info("Persisting dcrCoadd")
452 dataRef.put(results.dcrCoadds[subfilter], "dcrCoadd", subfilter=subfilter,
453 numSubfilters=self.config.dcrNumSubfilters)
454 if self.config.doNImage and results.dcrNImages is not None:
455 dataRef.put(results.dcrNImages[subfilter], "dcrCoadd_nImage", subfilter=subfilter,
456 numSubfilters=self.config.dcrNumSubfilters)
457
458 return results
459
460 @utils.inheritDoc(AssembleCoaddTask)
461 def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs):
462 """Load the previously-generated template coadd.
463
464 This can be removed entirely once we no longer support the Gen 2 butler.
465
466 Returns
467 -------
468 templateCoadd : `lsst.pipe.base.Struct`
469 Result struct with components:
470
471 - ``templateCoadd``: coadded exposure (`lsst.afw.image.ExposureF`)
472 """
473 templateCoadd = butlerQC.get(inputRefs.templateExposure)
474
475 return pipeBase.Struct(templateCoadd=templateCoadd)
476
477 def measureCoaddPsf(self, coaddExposure):
478 """Detect sources on the coadd exposure and measure the final PSF.
479
480 Parameters
481 ----------
482 coaddExposure : `lsst.afw.image.Exposure`
483 The final coadded exposure.
484 """
485 table = afwTable.SourceTable.make(self.schema)
486 detResults = self.detectPsfSources.run(table, coaddExposure, clearMask=False)
487 coaddSources = detResults.sources
488 self.measurePsfSources.run(
489 measCat=coaddSources,
490 exposure=coaddExposure
491 )
492 # Measure the PSF on the stacked subfilter coadds if possible.
493 # We should already have a decent estimate of the coadd PSF, however,
494 # so in case of any errors simply log them as a warning and use the
495 # default PSF.
496 try:
497 psfResults = self.measurePsf.run(coaddExposure, coaddSources)
498 except Exception as e:
499 self.log.warning("Unable to calculate PSF, using default coadd PSF: %s", e)
500 else:
501 coaddExposure.setPsf(psfResults.psf)
502
503 def prepareDcrInputs(self, templateCoadd, warpRefList, weightList):
504 """Prepare the DCR coadd by iterating through the visitInfo of the input warps.
505
506 Sets the property ``bufferSize``.
507
508 Parameters
509 ----------
510 templateCoadd : `lsst.afw.image.ExposureF`
511 The initial coadd exposure before accounting for DCR.
512 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
513 `lsst.daf.persistence.ButlerDataRef`
514 The data references to the input warped exposures.
515 weightList : `list` of `float`
516 The weight to give each input exposure in the coadd
517 Will be modified in place if ``doAirmassWeight`` is set.
518
519 Returns
520 -------
521 dcrModels : `lsst.pipe.tasks.DcrModel`
522 Best fit model of the true sky after correcting chromatic effects.
523
524 Raises
525 ------
526 NotImplementedError
527 If ``lambdaMin`` is missing from the Mapper class of the obs package being used.
528 """
529 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
530 filterLabel = templateCoadd.getFilterLabel()
531 tempExpName = self.getTempExpDatasetName(self.warpType)
532 dcrShifts = []
533 airmassDict = {}
534 angleDict = {}
535 psfSizeDict = {}
536 for visitNum, warpExpRef in enumerate(warpRefList):
537 if isinstance(warpExpRef, DeferredDatasetHandle):
538 # Gen 3 API
539 visitInfo = warpExpRef.get(component="visitInfo")
540 psf = warpExpRef.get(component="psf")
541 else:
542 # Gen 2 API. Delete this when Gen 2 retired
543 visitInfo = warpExpRef.get(tempExpName + "_visitInfo")
544 psf = warpExpRef.get(tempExpName).getPsf()
545 visit = warpExpRef.dataId["visit"]
546 # Just need a rough estimate; average positions are fine
547 psfAvgPos = psf.getAveragePosition()
548 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
549 airmass = visitInfo.getBoresightAirmass()
550 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
551 airmassDict[visit] = airmass
552 angleDict[visit] = parallacticAngle
553 psfSizeDict[visit] = psfSize
554 if self.config.doAirmassWeight:
555 weightList[visitNum] *= airmass
556 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
557 self.config.effectiveWavelength,
558 self.config.bandwidth,
559 self.config.dcrNumSubfilters))))
560 self.log.info("Selected airmasses:\n%s", airmassDict)
561 self.log.info("Selected parallactic angles:\n%s", angleDict)
562 self.log.info("Selected PSF sizes:\n%s", psfSizeDict)
563 self.bufferSize = int(np.ceil(np.max(dcrShifts)) + 1)
564 try:
565 psf = self.selectCoaddPsf(templateCoadd, warpRefList)
566 except Exception as e:
567 self.log.warning("Unable to calculate restricted PSF, using default coadd PSF: %s", e)
568 else:
569 psf = templateCoadd.getPsf()
570 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
571 self.config.dcrNumSubfilters,
572 effectiveWavelength=self.config.effectiveWavelength,
573 bandwidth=self.config.bandwidth,
574 filterLabel=filterLabel,
575 psf=psf)
576 return dcrModels
577
578 @timeMethod
579 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
580 supplementaryData=None):
581 """Assemble the coadd.
582
583 Requires additional inputs Struct ``supplementaryData`` to contain a
584 ``templateCoadd`` that serves as the model of the static sky.
585
586 Find artifacts and apply them to the warps' masks creating a list of
587 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane
588 Then pass these alternative masks to the base class's assemble method.
589
590 Divide the ``templateCoadd`` evenly between each subfilter of a
591 ``DcrModel`` as the starting best estimate of the true wavelength-
592 dependent sky. Forward model the ``DcrModel`` using the known
593 chromatic effects in each subfilter and calculate a convergence metric
594 based on how well the modeled template matches the input warps. If
595 the convergence has not yet reached the desired threshold, then shift
596 and stack the residual images to build a new ``DcrModel``. Apply
597 conditioning to prevent oscillating solutions between iterations or
598 between subfilters.
599
600 Once the ``DcrModel`` reaches convergence or the maximum number of
601 iterations has been reached, fill the metadata for each subfilter
602 image and make them proper ``coaddExposure``s.
603
604 Parameters
605 ----------
606 skyInfo : `lsst.pipe.base.Struct`
607 Patch geometry information, from getSkyInfo
608 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
609 `lsst.daf.persistence.ButlerDataRef`
610 The data references to the input warped exposures.
611 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
612 The image scalars correct for the zero point of the exposures.
613 weightList : `list` of `float`
614 The weight to give each input exposure in the coadd
615 supplementaryData : `lsst.pipe.base.Struct`
616 Result struct returned by ``makeSupplementaryData`` with components:
617
618 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`)
619
620 Returns
621 -------
622 result : `lsst.pipe.base.Struct`
623 Result struct with components:
624
625 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`)
626 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`)
627 - ``dcrCoadds``: `list` of coadded exposures for each subfilter
628 - ``dcrNImages``: `list` of exposure count images for each subfilter
629 """
630 minNumIter = self.config.minNumIter or self.config.dcrNumSubfilters
631 maxNumIter = self.config.maxNumIter or self.config.dcrNumSubfilters*3
632 templateCoadd = supplementaryData.templateCoadd
633 baseMask = templateCoadd.mask.clone()
634 # The variance plane is for each subfilter
635 # and should be proportionately lower than the full-band image
636 baseVariance = templateCoadd.variance.clone()
637 baseVariance /= self.config.dcrNumSubfilters
638 spanSetMaskList = self.findArtifacts(templateCoadd, warpRefList, imageScalerList)
639 # Note that the mask gets cleared in ``findArtifacts``, but we want to preserve the mask.
640 templateCoadd.setMask(baseMask)
641 badMaskPlanes = self.config.badMaskPlanes[:]
642 # Note that is important that we do not add "CLIPPED" to ``badMaskPlanes``
643 # This is because pixels in observations that are significantly affect by DCR
644 # are likely to have many pixels that are both "DETECTED" and "CLIPPED",
645 # but those are necessary to constrain the DCR model.
646 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
647
648 stats = self.prepareStats(mask=badPixelMask)
649 dcrModels = self.prepareDcrInputs(templateCoadd, warpRefList, weightList)
650 if self.config.doNImage:
651 dcrNImages, dcrWeights = self.calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
652 spanSetMaskList, stats.ctrl)
653 nImage = afwImage.ImageU(skyInfo.bbox)
654 # Note that this nImage will be a factor of dcrNumSubfilters higher than
655 # the nImage returned by assembleCoadd for most pixels. This is because each
656 # subfilter may have a different nImage, and fractional values are not allowed.
657 for dcrNImage in dcrNImages:
658 nImage += dcrNImage
659 else:
660 dcrNImages = None
661
662 subregionSize = geom.Extent2I(*self.config.subregionSize)
663 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1])
664 * ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
665 subIter = 0
666 for subBBox in self._subBBoxIter(skyInfo.bbox, subregionSize):
667 modelIter = 0
668 subIter += 1
669 self.log.info("Computing coadd over patch %s subregion %s of %s: %s",
670 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
671 dcrBBox = geom.Box2I(subBBox)
672 dcrBBox.grow(self.bufferSize)
673 dcrBBox.clip(dcrModels.bbox)
674 modelWeights = self.calculateModelWeights(dcrModels, dcrBBox)
675 subExposures = self.loadSubExposures(dcrBBox, stats.ctrl, warpRefList,
676 imageScalerList, spanSetMaskList)
677 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
678 warpRefList, weightList, stats.ctrl)
679 self.log.info("Initial convergence : %s", convergenceMetric)
680 convergenceList = [convergenceMetric]
681 gainList = []
682 convergenceCheck = 1.
683 refImage = templateCoadd.image
684 while (convergenceCheck > self.config.convergenceThreshold or modelIter <= minNumIter):
685 gain = self.calculateGain(convergenceList, gainList)
686 self.dcrAssembleSubregion(dcrModels, subExposures, subBBox, dcrBBox, warpRefList,
687 stats.ctrl, convergenceMetric, gain,
688 modelWeights, refImage, dcrWeights)
689 if self.config.useConvergence:
690 convergenceMetric = self.calculateConvergence(dcrModels, subExposures, subBBox,
691 warpRefList, weightList, stats.ctrl)
692 if convergenceMetric == 0:
693 self.log.warning("Coadd patch %s subregion %s had convergence metric of 0.0 which is "
694 "most likely due to there being no valid data in the region.",
695 skyInfo.patchInfo.getIndex(), subIter)
696 break
697 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
698 if (convergenceCheck < 0) & (modelIter > minNumIter):
699 self.log.warning("Coadd patch %s subregion %s diverged before reaching maximum "
700 "iterations or desired convergence improvement of %s."
701 " Divergence: %s",
702 skyInfo.patchInfo.getIndex(), subIter,
703 self.config.convergenceThreshold, convergenceCheck)
704 break
705 convergenceList.append(convergenceMetric)
706 if modelIter > maxNumIter:
707 if self.config.useConvergence:
708 self.log.warning("Coadd patch %s subregion %s reached maximum iterations "
709 "before reaching desired convergence improvement of %s."
710 " Final convergence improvement: %s",
711 skyInfo.patchInfo.getIndex(), subIter,
712 self.config.convergenceThreshold, convergenceCheck)
713 break
714
715 if self.config.useConvergence:
716 self.log.info("Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
717 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
718 modelIter += 1
719 else:
720 if self.config.useConvergence:
721 self.log.info("Coadd patch %s subregion %s finished with "
722 "convergence metric %s after %s iterations",
723 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
724 else:
725 self.log.info("Coadd patch %s subregion %s finished after %s iterations",
726 skyInfo.patchInfo.getIndex(), subIter, modelIter)
727 if self.config.useConvergence and convergenceMetric > 0:
728 self.log.info("Final convergence improvement was %.4f%% overall",
729 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
730
731 dcrCoadds = self.fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
732 calibration=self.scaleZeroPoint.getPhotoCalib(),
733 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
734 mask=baseMask,
735 variance=baseVariance)
736 coaddExposure = self.stackCoadd(dcrCoadds)
737 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
738 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
739
740 def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl):
741 """Calculate the number of exposures contributing to each subfilter.
742
743 Parameters
744 ----------
745 dcrModels : `lsst.pipe.tasks.DcrModel`
746 Best fit model of the true sky after correcting chromatic effects.
747 bbox : `lsst.geom.box.Box2I`
748 Bounding box of the patch to coadd.
749 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
750 `lsst.daf.persistence.ButlerDataRef`
751 The data references to the input warped exposures.
752 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
753 Each element of the `dict` contains the new mask plane name
754 (e.g. "CLIPPED and/or "NO_DATA") as the key,
755 and the list of SpanSets to apply to the mask.
757 Statistics control object for coadd
758
759 Returns
760 -------
761 dcrNImages : `list` of `lsst.afw.image.ImageU`
762 List of exposure count images for each subfilter
763 dcrWeights : `list` of `lsst.afw.image.ImageF`
764 Per-pixel weights for each subfilter.
765 Equal to 1/(number of unmasked images contributing to each pixel).
766 """
767 dcrNImages = [afwImage.ImageU(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
768 dcrWeights = [afwImage.ImageF(bbox) for subfilter in range(self.config.dcrNumSubfilters)]
769 tempExpName = self.getTempExpDatasetName(self.warpType)
770 for warpExpRef, altMaskSpans in zip(warpRefList, spanSetMaskList):
771 if isinstance(warpExpRef, DeferredDatasetHandle):
772 # Gen 3 API
773 exposure = warpExpRef.get(parameters={'bbox': bbox})
774 else:
775 # Gen 2 API. Delete this when Gen 2 retired
776 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
777 visitInfo = exposure.getInfo().getVisitInfo()
778 wcs = exposure.getInfo().getWcs()
779 mask = exposure.mask
780 if altMaskSpans is not None:
781 self.applyAltMaskPlanes(mask, altMaskSpans)
782 weightImage = np.zeros_like(exposure.image.array)
783 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
784 # The weights must be shifted in exactly the same way as the residuals,
785 # because they will be used as the denominator in the weighted average of residuals.
786 weightsGenerator = self.dcrResiduals(weightImage, visitInfo, wcs,
787 dcrModels.effectiveWavelength, dcrModels.bandwidth)
788 for shiftedWeights, dcrNImage, dcrWeight in zip(weightsGenerator, dcrNImages, dcrWeights):
789 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
790 dcrWeight.array += shiftedWeights
791 # Exclude any pixels that don't have at least one exposure contributing in all subfilters
792 weightsThreshold = 1.
793 goodPix = dcrWeights[0].array > weightsThreshold
794 for weights in dcrWeights[1:]:
795 goodPix = (weights.array > weightsThreshold) & goodPix
796 for subfilter in range(self.config.dcrNumSubfilters):
797 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
798 dcrWeights[subfilter].array[~goodPix] = 0.
799 dcrNImages[subfilter].array[~goodPix] = 0
800 return (dcrNImages, dcrWeights)
801
802 def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList,
803 statsCtrl, convergenceMetric,
804 gain, modelWeights, refImage, dcrWeights):
805 """Assemble the DCR coadd for a sub-region.
806
807 Build a DCR-matched template for each input exposure, then shift the
808 residuals according to the DCR in each subfilter.
809 Stack the shifted residuals and apply them as a correction to the
810 solution from the previous iteration.
811 Restrict the new model solutions from varying by more than a factor of
812 `modelClampFactor` from the last solution, and additionally restrict the
813 individual subfilter models from varying by more than a factor of
814 `frequencyClampFactor` from their average.
815 Finally, mitigate potentially oscillating solutions by averaging the new
816 solution with the solution from the previous iteration, weighted by
817 their convergence metric.
818
819 Parameters
820 ----------
821 dcrModels : `lsst.pipe.tasks.DcrModel`
822 Best fit model of the true sky after correcting chromatic effects.
823 subExposures : `dict` of `lsst.afw.image.ExposureF`
824 The pre-loaded exposures for the current subregion.
825 bbox : `lsst.geom.box.Box2I`
826 Bounding box of the subregion to coadd.
827 dcrBBox : `lsst.geom.box.Box2I`
828 Sub-region of the coadd which includes a buffer to allow for DCR.
829 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
830 `lsst.daf.persistence.ButlerDataRef`
831 The data references to the input warped exposures.
833 Statistics control object for coadd
834 convergenceMetric : `float`
835 Quality of fit metric for the matched templates of the input images.
836 gain : `float`, optional
837 Relative weight to give the new solution when updating the model.
838 modelWeights : `numpy.ndarray` or `float`
839 A 2D array of weight values that tapers smoothly to zero away from detected sources.
840 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
841 refImage : `lsst.afw.image.Image`
842 A reference image used to supply the default pixel values.
843 dcrWeights : `list` of `lsst.afw.image.Image`
844 Per-pixel weights for each subfilter.
845 Equal to 1/(number of unmasked images contributing to each pixel).
846 """
847 residualGeneratorList = []
848
849 for warpExpRef in warpRefList:
850 visit = warpExpRef.dataId["visit"]
851 exposure = subExposures[visit]
852 visitInfo = exposure.getInfo().getVisitInfo()
853 wcs = exposure.getInfo().getWcs()
854 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
855 order=self.config.imageInterpOrder,
856 splitSubfilters=self.config.splitSubfilters,
857 splitThreshold=self.config.splitThreshold,
858 amplifyModel=self.config.accelerateModel)
859 residual = exposure.image.array - templateImage.array
860 # Note that the variance plane here is used to store weights, not the actual variance
861 residual *= exposure.variance.array
862 # The residuals are stored as a list of generators.
863 # This allows the residual for a given subfilter and exposure to be created
864 # on the fly, instead of needing to store them all in memory.
865 residualGeneratorList.append(self.dcrResiduals(residual, visitInfo, wcs,
866 dcrModels.effectiveWavelength,
867 dcrModels.bandwidth))
868
869 dcrSubModelOut = self.newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
870 gain=gain,
871 modelWeights=modelWeights,
872 refImage=refImage,
873 dcrWeights=dcrWeights)
874 dcrModels.assign(dcrSubModelOut, bbox)
875
876 def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth):
877 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts.
878
879 Parameters
880 ----------
881 residual : `numpy.ndarray`
882 The residual masked image for one exposure,
883 after subtracting the matched template
884 visitInfo : `lsst.afw.image.VisitInfo`
885 Metadata for the exposure.
887 Coordinate system definition (wcs) for the exposure.
888
889 Yields
890 ------
891 residualImage : `numpy.ndarray`
892 The residual image for the next subfilter, shifted for DCR.
893 """
894 # Pre-calculate the spline-filtered residual image, so that step can be
895 # skipped in the shift calculation in `applyDcr`.
896 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
897 # Note that `splitSubfilters` is always turned off in the reverse direction.
898 # This option introduces additional blurring if applied to the residuals.
899 dcrShift = calculateDcr(visitInfo, wcs, effectiveWavelength, bandwidth, self.config.dcrNumSubfilters,
900 splitSubfilters=False)
901 for dcr in dcrShift:
902 yield applyDcr(filteredResidual, dcr, useInverse=True, splitSubfilters=False,
903 doPrefilter=False, order=self.config.imageInterpOrder)
904
905 def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
906 gain, modelWeights, refImage, dcrWeights):
907 """Calculate a new DcrModel from a set of image residuals.
908
909 Parameters
910 ----------
911 dcrModels : `lsst.pipe.tasks.DcrModel`
912 Current model of the true sky after correcting chromatic effects.
913 residualGeneratorList : `generator` of `numpy.ndarray`
914 The residual image for the next subfilter, shifted for DCR.
915 dcrBBox : `lsst.geom.box.Box2I`
916 Sub-region of the coadd which includes a buffer to allow for DCR.
918 Statistics control object for coadd
919 gain : `float`
920 Relative weight to give the new solution when updating the model.
921 modelWeights : `numpy.ndarray` or `float`
922 A 2D array of weight values that tapers smoothly to zero away from detected sources.
923 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
924 refImage : `lsst.afw.image.Image`
925 A reference image used to supply the default pixel values.
926 dcrWeights : `list` of `lsst.afw.image.Image`
927 Per-pixel weights for each subfilter.
928 Equal to 1/(number of unmasked images contributing to each pixel).
929
930 Returns
931 -------
932 dcrModel : `lsst.pipe.tasks.DcrModel`
933 New model of the true sky after correcting chromatic effects.
934 """
935 newModelImages = []
936 for subfilter, model in enumerate(dcrModels):
937 residualsList = [next(residualGenerator) for residualGenerator in residualGeneratorList]
938 residual = np.sum(residualsList, axis=0)
939 residual *= dcrWeights[subfilter][dcrBBox].array
940 # `MaskedImage`s only support in-place addition, so rename for readability
941 newModel = model[dcrBBox].clone()
942 newModel.array += residual
943 # Catch any invalid values
944 badPixels = ~np.isfinite(newModel.array)
945 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
946 if self.config.regularizeModelIterations > 0:
947 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
948 self.config.regularizeModelIterations,
949 self.config.regularizationWidth)
950 newModelImages.append(newModel)
951 if self.config.regularizeModelFrequency > 0:
952 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
953 self.config.regularizeModelFrequency,
954 self.config.regularizationWidth)
955 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
956 self.applyModelWeights(newModelImages, refImage[dcrBBox], modelWeights)
957 return DcrModel(newModelImages, dcrModels.filter, dcrModels.effectiveWavelength,
958 dcrModels.bandwidth, dcrModels.psf,
959 dcrModels.mask, dcrModels.variance)
960
961 def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl):
962 """Calculate a quality of fit metric for the matched templates.
963
964 Parameters
965 ----------
966 dcrModels : `lsst.pipe.tasks.DcrModel`
967 Best fit model of the true sky after correcting chromatic effects.
968 subExposures : `dict` of `lsst.afw.image.ExposureF`
969 The pre-loaded exposures for the current subregion.
970 bbox : `lsst.geom.box.Box2I`
971 Sub-region to coadd
972 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
973 `lsst.daf.persistence.ButlerDataRef`
974 The data references to the input warped exposures.
975 weightList : `list` of `float`
976 The weight to give each input exposure in the coadd
978 Statistics control object for coadd
979
980 Returns
981 -------
982 convergenceMetric : `float`
983 Quality of fit metric for all input exposures, within the sub-region
984 """
985 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
986 nSigma = 3.
987 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
988 bufferSize=self.bufferSize)
989 if np.max(significanceImage) == 0:
990 significanceImage += 1.
991 weight = 0
992 metric = 0.
993 metricList = {}
994 for warpExpRef, expWeight in zip(warpRefList, weightList):
995 visit = warpExpRef.dataId["visit"]
996 exposure = subExposures[visit][bbox]
997 singleMetric = self.calculateSingleConvergence(dcrModels, exposure, significanceImage, statsCtrl)
998 metric += singleMetric
999 metricList[visit] = singleMetric
1000 weight += 1.
1001 self.log.info("Individual metrics:\n%s", metricList)
1002 return 1.0 if weight == 0.0 else metric/weight
1003
1004 def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl):
1005 """Calculate a quality of fit metric for a single matched template.
1006
1007 Parameters
1008 ----------
1009 dcrModels : `lsst.pipe.tasks.DcrModel`
1010 Best fit model of the true sky after correcting chromatic effects.
1011 exposure : `lsst.afw.image.ExposureF`
1012 The input warped exposure to evaluate.
1013 significanceImage : `numpy.ndarray`
1014 Array of weights for each pixel corresponding to its significance
1015 for the convergence calculation.
1017 Statistics control object for coadd
1018
1019 Returns
1020 -------
1021 convergenceMetric : `float`
1022 Quality of fit metric for one exposure, within the sub-region.
1023 """
1024 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1025 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
1026 order=self.config.imageInterpOrder,
1027 splitSubfilters=self.config.splitSubfilters,
1028 splitThreshold=self.config.splitThreshold,
1029 amplifyModel=self.config.accelerateModel)
1030 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
1031 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
1032
1033 finitePixels = np.isfinite(diffVals)
1034 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
1035 convergeMaskPixels = exposure.mask.array & convergeMask > 0
1036 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
1037 if np.sum(usePixels) == 0:
1038 metric = 0.
1039 else:
1040 diffUse = diffVals[usePixels]
1041 refUse = refVals[usePixels]
1042 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
1043 return metric
1044
1045 def stackCoadd(self, dcrCoadds):
1046 """Add a list of sub-band coadds together.
1047
1048 Parameters
1049 ----------
1050 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1051 A list of coadd exposures, each exposure containing
1052 the model for one subfilter.
1053
1054 Returns
1055 -------
1056 coaddExposure : `lsst.afw.image.ExposureF`
1057 A single coadd exposure that is the sum of the sub-bands.
1058 """
1059 coaddExposure = dcrCoadds[0].clone()
1060 for coadd in dcrCoadds[1:]:
1061 coaddExposure.maskedImage += coadd.maskedImage
1062 return coaddExposure
1063
1064 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
1065 mask=None, variance=None):
1066 """Create a list of coadd exposures from a list of masked images.
1067
1068 Parameters
1069 ----------
1070 dcrModels : `lsst.pipe.tasks.DcrModel`
1071 Best fit model of the true sky after correcting chromatic effects.
1072 skyInfo : `lsst.pipe.base.Struct`
1073 Patch geometry information, from getSkyInfo
1074 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1075 `lsst.daf.persistence.ButlerDataRef`
1076 The data references to the input warped exposures.
1077 weightList : `list` of `float`
1078 The weight to give each input exposure in the coadd
1079 calibration : `lsst.afw.Image.PhotoCalib`, optional
1080 Scale factor to set the photometric calibration of an exposure.
1081 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional
1082 A record of the observations that are included in the coadd.
1083 mask : `lsst.afw.image.Mask`, optional
1084 Optional mask to override the values in the final coadd.
1085 variance : `lsst.afw.image.Image`, optional
1086 Optional variance plane to override the values in the final coadd.
1087
1088 Returns
1089 -------
1090 dcrCoadds : `list` of `lsst.afw.image.ExposureF`
1091 A list of coadd exposures, each exposure containing
1092 the model for one subfilter.
1093 """
1094 dcrCoadds = []
1095 refModel = dcrModels.getReferenceImage()
1096 for model in dcrModels:
1097 if self.config.accelerateModel > 1:
1098 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
1099 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
1100 if calibration is not None:
1101 coaddExposure.setPhotoCalib(calibration)
1102 if coaddInputs is not None:
1103 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
1104 # Set the metadata for the coadd, including PSF and aperture corrections.
1105 self.assembleMetadata(coaddExposure, warpRefList, weightList)
1106 # Overwrite the PSF
1107 coaddExposure.setPsf(dcrModels.psf)
1108 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
1109 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
1110 maskedImage.image = model
1111 maskedImage.mask = dcrModels.mask
1112 maskedImage.variance = dcrModels.variance
1113 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
1114 coaddExposure.setPhotoCalib(self.scaleZeroPoint.getPhotoCalib())
1115 if mask is not None:
1116 coaddExposure.setMask(mask)
1117 if variance is not None:
1118 coaddExposure.setVariance(variance)
1119 dcrCoadds.append(coaddExposure)
1120 return dcrCoadds
1121
1122 def calculateGain(self, convergenceList, gainList):
1123 """Calculate the gain to use for the current iteration.
1124
1125 After calculating a new DcrModel, each value is averaged with the
1126 value in the corresponding pixel from the previous iteration. This
1127 reduces oscillating solutions that iterative techniques are plagued by,
1128 and speeds convergence. By far the biggest changes to the model
1129 happen in the first couple iterations, so we can also use a more
1130 aggressive gain later when the model is changing slowly.
1131
1132 Parameters
1133 ----------
1134 convergenceList : `list` of `float`
1135 The quality of fit metric from each previous iteration.
1136 gainList : `list` of `float`
1137 The gains used in each previous iteration: appended with the new
1138 gain value.
1139 Gains are numbers between ``self.config.baseGain`` and 1.
1140
1141 Returns
1142 -------
1143 gain : `float`
1144 Relative weight to give the new solution when updating the model.
1145 A value of 1.0 gives equal weight to both solutions.
1146
1147 Raises
1148 ------
1149 ValueError
1150 If ``len(convergenceList) != len(gainList)+1``.
1151 """
1152 nIter = len(convergenceList)
1153 if nIter != len(gainList) + 1:
1154 raise ValueError("convergenceList (%d) must be one element longer than gainList (%d)."
1155 % (len(convergenceList), len(gainList)))
1156
1157 if self.config.baseGain is None:
1158 # If ``baseGain`` is not set, calculate it from the number of DCR subfilters
1159 # The more subfilters being modeled, the lower the gain should be.
1160 baseGain = 1./(self.config.dcrNumSubfilters - 1)
1161 else:
1162 baseGain = self.config.baseGain
1163
1164 if self.config.useProgressiveGain and nIter > 2:
1165 # To calculate the best gain to use, compare the past gains that have been used
1166 # with the resulting convergences to estimate the best gain to use.
1167 # Algorithmically, this is a Kalman filter.
1168 # If forward modeling proceeds perfectly, the convergence metric should
1169 # asymptotically approach a final value.
1170 # We can estimate that value from the measured changes in convergence
1171 # weighted by the gains used in each previous iteration.
1172 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
1173 for i in range(nIter - 1)]
1174 # The convergence metric is strictly positive, so if the estimated final convergence is
1175 # less than zero, force it to zero.
1176 estFinalConv = np.array(estFinalConv)
1177 estFinalConv[estFinalConv < 0] = 0
1178 # Because the estimate may slowly change over time, only use the most recent measurements.
1179 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
1180 lastGain = gainList[-1]
1181 lastConv = convergenceList[-2]
1182 newConv = convergenceList[-1]
1183 # The predicted convergence is the value we would get if the new model calculated
1184 # in the previous iteration was perfect. Recall that the updated model that is
1185 # actually used is the gain-weighted average of the new and old model,
1186 # so the convergence would be similarly weighted.
1187 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1188 # If the measured and predicted convergence are very close, that indicates
1189 # that our forward model is accurate and we can use a more aggressive gain
1190 # If the measured convergence is significantly worse (or better!) than predicted,
1191 # that indicates that the model is not converging as expected and
1192 # we should use a more conservative gain.
1193 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1194 newGain = 1 - abs(delta)
1195 # Average the gains to prevent oscillating solutions.
1196 newGain = (newGain + lastGain)/2.
1197 gain = max(baseGain, newGain)
1198 else:
1199 gain = baseGain
1200 gainList.append(gain)
1201 return gain
1202
1203 def calculateModelWeights(self, dcrModels, dcrBBox):
1204 """Build an array that smoothly tapers to 0 away from detected sources.
1205
1206 Parameters
1207 ----------
1208 dcrModels : `lsst.pipe.tasks.DcrModel`
1209 Best fit model of the true sky after correcting chromatic effects.
1210 dcrBBox : `lsst.geom.box.Box2I`
1211 Sub-region of the coadd which includes a buffer to allow for DCR.
1212
1213 Returns
1214 -------
1215 weights : `numpy.ndarray` or `float`
1216 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1217 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1218
1219 Raises
1220 ------
1221 ValueError
1222 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative.
1223 """
1224 if not self.config.useModelWeights:
1225 return 1.0
1226 if self.config.modelWeightsWidth < 0:
1227 raise ValueError("modelWeightsWidth must not be negative if useModelWeights is set")
1228 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1229 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1230 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1231 weights[convergeMaskPixels] = 1.
1232 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1233 weights /= np.max(weights)
1234 return weights
1235
1236 def applyModelWeights(self, modelImages, refImage, modelWeights):
1237 """Smoothly replace model pixel values with those from a
1238 reference at locations away from detected sources.
1239
1240 Parameters
1241 ----------
1242 modelImages : `list` of `lsst.afw.image.Image`
1243 The new DCR model images from the current iteration.
1244 The values will be modified in place.
1245 refImage : `lsst.afw.image.MaskedImage`
1246 A reference image used to supply the default pixel values.
1247 modelWeights : `numpy.ndarray` or `float`
1248 A 2D array of weight values that tapers smoothly to zero away from detected sources.
1249 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False.
1250 """
1251 if self.config.useModelWeights:
1252 for model in modelImages:
1253 model.array *= modelWeights
1254 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1255
1256 def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList):
1257 """Pre-load sub-regions of a list of exposures.
1258
1259 Parameters
1260 ----------
1261 bbox : `lsst.geom.box.Box2I`
1262 Sub-region to coadd
1264 Statistics control object for coadd
1265 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1266 `lsst.daf.persistence.ButlerDataRef`
1267 The data references to the input warped exposures.
1268 imageScalerList : `list` of `lsst.pipe.task.ImageScaler`
1269 The image scalars correct for the zero point of the exposures.
1270 spanSetMaskList : `list` of `dict` containing spanSet lists, or None
1271 Each element is dict with keys = mask plane name to add the spans to
1272
1273 Returns
1274 -------
1275 subExposures : `dict`
1276 The `dict` keys are the visit IDs,
1277 and the values are `lsst.afw.image.ExposureF`
1278 The pre-loaded exposures for the current subregion.
1279 The variance plane contains weights, and not the variance
1280 """
1281 tempExpName = self.getTempExpDatasetName(self.warpType)
1282 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1283 subExposures = {}
1284 for warpExpRef, imageScaler, altMaskSpans in zipIterables:
1285 if isinstance(warpExpRef, DeferredDatasetHandle):
1286 exposure = warpExpRef.get(parameters={'bbox': bbox})
1287 else:
1288 exposure = warpExpRef.get(tempExpName + "_sub", bbox=bbox)
1289 visit = warpExpRef.dataId["visit"]
1290 if altMaskSpans is not None:
1291 self.applyAltMaskPlanes(exposure.mask, altMaskSpans)
1292 imageScaler.scaleMaskedImage(exposure.maskedImage)
1293 # Note that the variance plane here is used to store weights, not the actual variance
1294 exposure.variance.array[:, :] = 0.
1295 # Set the weight of unmasked pixels to 1.
1296 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1297 # Set the image value of masked pixels to zero.
1298 # This eliminates needing the mask plane when stacking images in ``newModelFromResidual``
1299 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1300 subExposures[visit] = exposure
1301 return subExposures
1302
1303 def selectCoaddPsf(self, templateCoadd, warpRefList):
1304 """Compute the PSF of the coadd from the exposures with the best seeing.
1305
1306 Parameters
1307 ----------
1308 templateCoadd : `lsst.afw.image.ExposureF`
1309 The initial coadd exposure before accounting for DCR.
1310 warpRefList : `list` of `lsst.daf.butler.DeferredDatasetHandle` or
1311 `lsst.daf.persistence.ButlerDataRef`
1312 The data references to the input warped exposures.
1313
1314 Returns
1315 -------
1317 The average PSF of the input exposures with the best seeing.
1318 """
1319 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1320 tempExpName = self.getTempExpDatasetName(self.warpType)
1321 # Note: ``ccds`` is a `lsst.afw.table.ExposureCatalog` with one entry per ccd and per visit
1322 # If there are multiple ccds, it will have that many times more elements than ``warpExpRef``
1323 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1324 templatePsf = templateCoadd.getPsf()
1325 # Just need a rough estimate; average positions are fine
1326 templateAvgPos = templatePsf.getAveragePosition()
1327 psfRefSize = templatePsf.computeShape(templateAvgPos).getDeterminantRadius()*sigma2fwhm
1328 psfSizes = np.zeros(len(ccds))
1329 ccdVisits = np.array(ccds["visit"])
1330 for warpExpRef in warpRefList:
1331 if isinstance(warpExpRef, DeferredDatasetHandle):
1332 # Gen 3 API
1333 psf = warpExpRef.get(component="psf")
1334 else:
1335 # Gen 2 API. Delete this when Gen 2 retired
1336 psf = warpExpRef.get(tempExpName).getPsf()
1337 visit = warpExpRef.dataId["visit"]
1338 psfAvgPos = psf.getAveragePosition()
1339 psfSize = psf.computeShape(psfAvgPos).getDeterminantRadius()*sigma2fwhm
1340 psfSizes[ccdVisits == visit] = psfSize
1341 # Note that the input PSFs include DCR, which should be absent from the DcrCoadd
1342 # The selected PSFs are those that have a FWHM less than or equal to the smaller
1343 # of the mean or median FWHM of the input exposures.
1344 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1345 goodPsfs = psfSizes <= sizeThreshold
1346 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1347 self.config.coaddPsf.makeControl())
1348 return psf
def makeSupplementaryDataGen3(self, butlerQC, inputRefs, outputRefs)
def makeSkyInfo(skyMap, tractId, patchId)
Definition: coaddBase.py:289
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def calculateGain(self, convergenceList, gainList)
def calculateModelWeights(self, dcrModels, dcrBBox)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def dcrResiduals(self, residual, visitInfo, wcs, effectiveWavelength, bandwidth)