25 from scipy
import ndimage
33 import lsst.pex.config
as pexConfig
35 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf
import MeasurePsfTask
38 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
42 dcrNumSubfilters = pexConfig.Field(
44 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
47 maxNumIter = pexConfig.Field(
50 doc=
"Maximum number of iterations of forward modeling.",
53 minNumIter = pexConfig.Field(
56 doc=
"Minimum number of iterations of forward modeling.",
59 convergenceThreshold = pexConfig.Field(
61 doc=
"Target relative change in convergence between iterations of forward modeling.",
64 useConvergence = pexConfig.Field(
66 doc=
"Use convergence test as a forward modeling end condition?" 67 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
70 baseGain = pexConfig.Field(
73 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 74 "A value of 1.0 gives equal weight to both solutions." 75 "Small values imply slower convergence of the solution, but can " 76 "help prevent overshooting and failures in the fit." 77 "If ``baseGain`` is None, a conservative gain " 78 "will be calculated from the number of subfilters. ",
81 useProgressiveGain = pexConfig.Field(
83 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 84 "When calculating the next gain, we use up to 5 previous gains and convergence values." 85 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
88 doAirmassWeight = pexConfig.Field(
90 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
93 modelWeightsWidth = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 useModelWeights = pexConfig.Field(
100 doc=
"Width of the region around detected sources to include in the DcrModel.",
103 splitSubfilters = pexConfig.Field(
105 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 106 "Instead of at the midpoint",
109 splitThreshold = pexConfig.Field(
111 doc=
"Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 112 "Set to 0 to always split the subfilters.",
115 regularizeModelIterations = pexConfig.Field(
117 doc=
"Maximum relative change of the model allowed between iterations." 118 "Set to zero to disable.",
121 regularizeModelFrequency = pexConfig.Field(
123 doc=
"Maximum relative change of the model allowed between subfilters." 124 "Set to zero to disable.",
127 convergenceMaskPlanes = pexConfig.ListField(
129 default=[
"DETECTED"],
130 doc=
"Mask planes to use to calculate convergence." 132 regularizationWidth = pexConfig.Field(
135 doc=
"Minimum radius of a region to include in regularization, in pixels." 137 imageInterpOrder = pexConfig.Field(
139 doc=
"The order of the spline interpolation used to shift the image plane.",
142 accelerateModel = pexConfig.Field(
144 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
147 doCalculatePsf = pexConfig.Field(
149 doc=
"Set to detect stars and recalculate the PSF from the final coadd." 150 "Otherwise the PSF is estimated from a selection of the best input exposures",
153 detectPsfSources = pexConfig.ConfigurableField(
154 target=measAlg.SourceDetectionTask,
155 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
157 measurePsfSources = pexConfig.ConfigurableField(
158 target=SingleFrameMeasurementTask,
159 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 161 measurePsf = pexConfig.ConfigurableField(
162 target=MeasurePsfTask,
163 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
167 CompareWarpAssembleCoaddConfig.setDefaults(self)
186 self.
measurePsf.starSelector[
"objectSize"].doFluxLimit =
False 190 """Assemble DCR coadded images from a set of warps. 195 The number of pixels to grow each subregion by to allow for DCR. 199 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 200 Warps (also called coadded temporary exposures), including the effects of 201 Differential Chromatic Refraction (DCR). 202 For full details of the mathematics and algorithm, please see 203 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 205 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 206 each subfilter used in the iterative calculation. 207 It begins by dividing the bandpass-defining filter into N equal bandwidth 208 "subfilters", and divides the flux in each pixel from an initial coadd 209 equally into each as a "dcrModel". Because the airmass and parallactic 210 angle of each individual exposure is known, we can calculate the shift 211 relative to the center of the band in each subfilter due to DCR. For each 212 exposure we apply this shift as a linear transformation to the dcrModels 213 and stack the results to produce a DCR-matched exposure. The matched 214 exposures are subtracted from the input exposures to produce a set of 215 residual images, and these residuals are reverse shifted for each 216 exposures' subfilters and stacked. The shifted and stacked residuals are 217 added to the dcrModels to produce a new estimate of the flux in each pixel 218 within each subfilter. The dcrModels are solved for iteratively, which 219 continues until the solution from a new iteration improves by less than 220 a set percentage, or a maximum number of iterations is reached. 221 Two forms of regularization are employed to reduce unphysical results. 222 First, the new solution is averaged with the solution from the previous 223 iteration, which mitigates oscillating solutions where the model 224 overshoots with alternating very high and low values. 225 Second, a common degeneracy when the data have a limited range of airmass or 226 parallactic angle values is for one subfilter to be fit with very low or 227 negative values, while another subfilter is fit with very high values. This 228 typically appears in the form of holes next to sources in one subfilter, 229 and corresponding extended wings in another. Because each subfilter has 230 a narrow bandwidth we assume that physical sources that are above the noise 231 level will not vary in flux by more than a factor of `frequencyClampFactor` 232 between subfilters, and pixels that have flux deviations larger than that 233 factor will have the excess flux distributed evenly among all subfilters. 234 If `splitSubfilters` is set, then each subfilter will be further sub- 235 divided during the forward modeling step (only). This approximates using 236 a higher number of subfilters that may be necessary for high airmass 237 observations, but does not increase the number of free parameters in the 238 fit. This is needed when there are high airmass observations which would 239 otherwise have significant DCR even within a subfilter. Because calculating 240 the shifted images takes most of the time, splitting the subfilters is 241 turned off by way of the `splitThreshold` option for low-airmass 242 observations that do not suffer from DCR within a subfilter. 245 ConfigClass = DcrAssembleCoaddConfig
246 _DefaultName =
"dcrAssembleCoadd" 250 if self.config.doCalculatePsf:
251 self.
schema = afwTable.SourceTable.makeMinimalSchema()
252 self.makeSubtask(
"detectPsfSources", schema=self.
schema)
253 self.makeSubtask(
"measurePsfSources", schema=self.
schema)
254 self.makeSubtask(
"measurePsf", schema=self.
schema)
257 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258 """Assemble a coadd from a set of warps. 260 Coadd a set of Warps. Compute weights to be applied to each Warp and 261 find scalings to match the photometric zeropoint to a reference Warp. 262 Assemble the Warps using run method. 263 Forward model chromatic effects across multiple subfilters, 264 and subtract from the input Warps to build sets of residuals. 265 Use the residuals to construct a new ``DcrModel`` for each subfilter, 266 and iterate until the model converges. 267 Interpolate over NaNs and optionally write the coadd to disk. 268 Return the coadded exposure. 272 dataRef : `lsst.daf.persistence.ButlerDataRef` 273 Data reference defining the patch for coaddition and the 275 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 276 List of data references to warps. Data to be coadded will be 277 selected from this list based on overlap with the patch defined by 282 results : `lsst.pipe.base.Struct` 283 The Struct contains the following fields: 285 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 286 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 287 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 288 - ``dcrNImages``: `list` of exposure count images for each subfilter 290 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
291 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
293 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294 warpRefList=warpRefList)
297 self.log.warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
298 skyInfo.patchInfo.getIndex())
300 for subfilter
in range(self.config.dcrNumSubfilters):
302 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
303 AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
304 if self.config.doWrite:
305 self.log.info(
"Persisting dcrCoadd")
306 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
307 numSubfilters=self.config.dcrNumSubfilters)
308 if self.config.doNImage
and results.dcrNImages
is not None:
309 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
310 numSubfilters=self.config.dcrNumSubfilters)
315 """Interpolate over missing data and mask bright stars. 317 Also detect sources on the coadd exposure and measure the final PSF, 318 if ``doCalculatePsf`` is set. 322 coaddExposure : `lsst.afw.image.Exposure` 323 The final coadded exposure. 324 dataRef : `lsst.daf.persistence.ButlerDataRef` 325 Data reference defining the patch for coaddition and the 330 if self.config.doCalculatePsf:
331 expId = dataRef.get(
"dcrCoaddId")
332 table = afwTable.SourceTable.make(self.
schema)
333 detResults = self.detectPsfSources.
run(table, coaddExposure, expId, clearMask=
False)
334 coaddSources = detResults.sources
335 self.measurePsfSources.
run(
336 measCat=coaddSources,
337 exposure=coaddExposure,
345 psfResults = self.measurePsf.
run(coaddExposure, coaddSources, expId=expId)
346 except Exception
as e:
347 self.log.warn(
"Unable to calculate PSF, using default coadd PSF: %s" % e)
349 coaddExposure.setPsf(psfResults.psf)
352 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 354 Sets the property ``bufferSize``. 358 templateCoadd : `lsst.afw.image.ExposureF` 359 The initial coadd exposure before accounting for DCR. 360 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 361 The data references to the input warped exposures. 362 weightList : `list` of `float` 363 The weight to give each input exposure in the coadd 364 Will be modified in place if ``doAirmassWeight`` is set. 368 dcrModels : `lsst.pipe.tasks.DcrModel` 369 Best fit model of the true sky after correcting chromatic effects. 374 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 376 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
377 filterInfo = templateCoadd.getFilter()
378 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
379 raise NotImplementedError(
"No minimum/maximum wavelength information found" 380 " in the filter definition! Please add lambdaMin and lambdaMax" 381 " to the Mapper class in your obs package.")
387 for visitNum, warpExpRef
in enumerate(warpRefList):
388 visitInfo = warpExpRef.get(tempExpName +
"_visitInfo")
389 visit = warpExpRef.dataId[
"visit"]
390 psf = warpExpRef.get(tempExpName).getPsf()
391 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
392 airmass = visitInfo.getBoresightAirmass()
393 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
394 airmassDict[visit] = airmass
395 angleDict[visit] = parallacticAngle
396 psfSizeDict[visit] = psfSize
397 if self.config.doAirmassWeight:
398 weightList[visitNum] *= airmass
399 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
400 filterInfo, self.config.dcrNumSubfilters))))
401 self.log.info(
"Selected airmasses:\n%s", airmassDict)
402 self.log.info(
"Selected parallactic angles:\n%s", angleDict)
403 self.log.info(
"Selected PSF sizes:\n%s", psfSizeDict)
406 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
407 self.config.dcrNumSubfilters,
408 filterInfo=filterInfo,
412 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
413 supplementaryData=None):
414 """Assemble the coadd. 416 Requires additional inputs Struct ``supplementaryData`` to contain a 417 ``templateCoadd`` that serves as the model of the static sky. 419 Find artifacts and apply them to the warps' masks creating a list of 420 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 421 Then pass these alternative masks to the base class's assemble method. 423 Divide the ``templateCoadd`` evenly between each subfilter of a 424 ``DcrModel`` as the starting best estimate of the true wavelength- 425 dependent sky. Forward model the ``DcrModel`` using the known 426 chromatic effects in each subfilter and calculate a convergence metric 427 based on how well the modeled template matches the input warps. If 428 the convergence has not yet reached the desired threshold, then shift 429 and stack the residual images to build a new ``DcrModel``. Apply 430 conditioning to prevent oscillating solutions between iterations or 433 Once the ``DcrModel`` reaches convergence or the maximum number of 434 iterations has been reached, fill the metadata for each subfilter 435 image and make them proper ``coaddExposure``s. 439 skyInfo : `lsst.pipe.base.Struct` 440 Patch geometry information, from getSkyInfo 441 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 442 The data references to the input warped exposures. 443 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 444 The image scalars correct for the zero point of the exposures. 445 weightList : `list` of `float` 446 The weight to give each input exposure in the coadd 447 supplementaryData : `lsst.pipe.base.Struct` 448 Result struct returned by ``makeSupplementaryData`` with components: 450 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 454 result : `lsst.pipe.base.Struct` 455 Result struct with components: 457 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 458 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 459 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 460 - ``dcrNImages``: `list` of exposure count images for each subfilter 462 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
463 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
464 templateCoadd = supplementaryData.templateCoadd
465 baseMask = templateCoadd.mask.clone()
468 baseVariance = templateCoadd.variance.clone()
469 baseVariance /= self.config.dcrNumSubfilters
470 spanSetMaskList = self.
findArtifacts(templateCoadd, warpRefList, imageScalerList)
472 templateCoadd.setMask(baseMask)
473 badMaskPlanes = self.config.badMaskPlanes[:]
478 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
482 if self.config.doNImage:
483 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
484 spanSetMaskList, stats.ctrl)
485 nImage = afwImage.ImageU(skyInfo.bbox)
489 for dcrNImage
in dcrNImages:
495 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
496 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
498 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
501 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
502 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
505 dcrBBox.clip(dcrModels.bbox)
508 imageScalerList, spanSetMaskList)
510 warpRefList, weightList, stats.ctrl)
511 self.log.info(
"Initial convergence : %s", convergenceMetric)
512 convergenceList = [convergenceMetric]
514 convergenceCheck = 1.
515 refImage = templateCoadd.image
516 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
519 stats.ctrl, convergenceMetric, gain,
520 modelWeights, refImage, dcrWeights)
521 if self.config.useConvergence:
523 warpRefList, weightList, stats.ctrl)
524 if convergenceMetric == 0:
525 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 526 "most likely due to there being no valid data in the region.",
527 skyInfo.patchInfo.getIndex(), subIter)
529 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
530 if (convergenceCheck < 0) & (modelIter > minNumIter):
531 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 532 "iterations or desired convergence improvement of %s." 534 skyInfo.patchInfo.getIndex(), subIter,
535 self.config.convergenceThreshold, convergenceCheck)
537 convergenceList.append(convergenceMetric)
538 if modelIter > maxNumIter:
539 if self.config.useConvergence:
540 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 541 "before reaching desired convergence improvement of %s." 542 " Final convergence improvement: %s",
543 skyInfo.patchInfo.getIndex(), subIter,
544 self.config.convergenceThreshold, convergenceCheck)
547 if self.config.useConvergence:
548 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
549 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
552 if self.config.useConvergence:
553 self.log.info(
"Coadd patch %s subregion %s finished with " 554 "convergence metric %s after %s iterations",
555 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
557 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
558 skyInfo.patchInfo.getIndex(), subIter, modelIter)
559 if self.config.useConvergence
and convergenceMetric > 0:
560 self.log.info(
"Final convergence improvement was %.4f%% overall",
561 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
563 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
564 calibration=self.scaleZeroPoint.getPhotoCalib(),
565 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
567 variance=baseVariance)
569 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
570 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
573 """Calculate the number of exposures contributing to each subfilter. 577 dcrModels : `lsst.pipe.tasks.DcrModel` 578 Best fit model of the true sky after correcting chromatic effects. 579 bbox : `lsst.geom.box.Box2I` 580 Bounding box of the patch to coadd. 581 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 582 The data references to the input warped exposures. 583 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 584 Each element of the `dict` contains the new mask plane name 585 (e.g. "CLIPPED and/or "NO_DATA") as the key, 586 and the list of SpanSets to apply to the mask. 587 statsCtrl : `lsst.afw.math.StatisticsControl` 588 Statistics control object for coadd 592 dcrNImages : `list` of `lsst.afw.image.ImageU` 593 List of exposure count images for each subfilter 594 dcrWeights : `list` of `lsst.afw.image.ImageF` 595 Per-pixel weights for each subfilter. 596 Equal to 1/(number of unmasked images contributing to each pixel). 598 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
599 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
601 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
602 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
603 visitInfo = exposure.getInfo().getVisitInfo()
604 wcs = exposure.getInfo().getWcs()
606 if altMaskSpans
is not None:
608 weightImage = np.zeros_like(exposure.image.array)
609 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
612 weightsGenerator = self.
dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
613 for shiftedWeights, dcrNImage, dcrWeight
in zip(weightsGenerator, dcrNImages, dcrWeights):
614 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
615 dcrWeight.array += shiftedWeights
617 weightsThreshold = 1.
618 goodPix = dcrWeights[0].array > weightsThreshold
619 for weights
in dcrWeights[1:]:
620 goodPix = (weights.array > weightsThreshold) & goodPix
621 for subfilter
in range(self.config.dcrNumSubfilters):
622 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
623 dcrWeights[subfilter].array[~goodPix] = 0.
624 dcrNImages[subfilter].array[~goodPix] = 0
625 return (dcrNImages, dcrWeights)
628 statsCtrl, convergenceMetric,
629 gain, modelWeights, refImage, dcrWeights):
630 """Assemble the DCR coadd for a sub-region. 632 Build a DCR-matched template for each input exposure, then shift the 633 residuals according to the DCR in each subfilter. 634 Stack the shifted residuals and apply them as a correction to the 635 solution from the previous iteration. 636 Restrict the new model solutions from varying by more than a factor of 637 `modelClampFactor` from the last solution, and additionally restrict the 638 individual subfilter models from varying by more than a factor of 639 `frequencyClampFactor` from their average. 640 Finally, mitigate potentially oscillating solutions by averaging the new 641 solution with the solution from the previous iteration, weighted by 642 their convergence metric. 646 dcrModels : `lsst.pipe.tasks.DcrModel` 647 Best fit model of the true sky after correcting chromatic effects. 648 subExposures : `dict` of `lsst.afw.image.ExposureF` 649 The pre-loaded exposures for the current subregion. 650 bbox : `lsst.geom.box.Box2I` 651 Bounding box of the subregion to coadd. 652 dcrBBox : `lsst.geom.box.Box2I` 653 Sub-region of the coadd which includes a buffer to allow for DCR. 654 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 655 The data references to the input warped exposures. 656 statsCtrl : `lsst.afw.math.StatisticsControl` 657 Statistics control object for coadd 658 convergenceMetric : `float` 659 Quality of fit metric for the matched templates of the input images. 660 gain : `float`, optional 661 Relative weight to give the new solution when updating the model. 662 modelWeights : `numpy.ndarray` or `float` 663 A 2D array of weight values that tapers smoothly to zero away from detected sources. 664 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 665 refImage : `lsst.afw.image.Image` 666 A reference image used to supply the default pixel values. 667 dcrWeights : `list` of `lsst.afw.image.Image` 668 Per-pixel weights for each subfilter. 669 Equal to 1/(number of unmasked images contributing to each pixel). 671 residualGeneratorList = []
673 for warpExpRef
in warpRefList:
674 exposure = subExposures[warpExpRef.dataId[
"visit"]]
675 visitInfo = exposure.getInfo().getVisitInfo()
676 wcs = exposure.getInfo().getWcs()
677 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
678 order=self.config.imageInterpOrder,
679 splitSubfilters=self.config.splitSubfilters,
680 splitThreshold=self.config.splitThreshold,
681 amplifyModel=self.config.accelerateModel)
682 residual = exposure.image.array - templateImage.array
684 residual *= exposure.variance.array
688 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
690 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
692 modelWeights=modelWeights,
694 dcrWeights=dcrWeights)
695 dcrModels.assign(dcrSubModelOut, bbox)
698 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 702 residual : `numpy.ndarray` 703 The residual masked image for one exposure, 704 after subtracting the matched template 705 visitInfo : `lsst.afw.image.VisitInfo` 706 Metadata for the exposure. 707 wcs : `lsst.afw.geom.SkyWcs` 708 Coordinate system definition (wcs) for the exposure. 709 filterInfo : `lsst.afw.image.Filter` 710 The filter definition, set in the current instruments' obs package. 711 Required for any calculation of DCR, including making matched templates. 715 residualImage : `numpy.ndarray` 716 The residual image for the next subfilter, shifted for DCR. 720 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
723 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
724 splitSubfilters=
False)
726 yield applyDcr(filteredResidual, dcr, useInverse=
True, splitSubfilters=
False,
727 doPrefilter=
False, order=self.config.imageInterpOrder)
730 gain, modelWeights, refImage, dcrWeights):
731 """Calculate a new DcrModel from a set of image residuals. 735 dcrModels : `lsst.pipe.tasks.DcrModel` 736 Current model of the true sky after correcting chromatic effects. 737 residualGeneratorList : `generator` of `numpy.ndarray` 738 The residual image for the next subfilter, shifted for DCR. 739 dcrBBox : `lsst.geom.box.Box2I` 740 Sub-region of the coadd which includes a buffer to allow for DCR. 741 statsCtrl : `lsst.afw.math.StatisticsControl` 742 Statistics control object for coadd 744 Relative weight to give the new solution when updating the model. 745 modelWeights : `numpy.ndarray` or `float` 746 A 2D array of weight values that tapers smoothly to zero away from detected sources. 747 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 748 refImage : `lsst.afw.image.Image` 749 A reference image used to supply the default pixel values. 750 dcrWeights : `list` of `lsst.afw.image.Image` 751 Per-pixel weights for each subfilter. 752 Equal to 1/(number of unmasked images contributing to each pixel). 756 dcrModel : `lsst.pipe.tasks.DcrModel` 757 New model of the true sky after correcting chromatic effects. 760 for subfilter, model
in enumerate(dcrModels):
761 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
762 residual = np.sum(residualsList, axis=0)
763 residual *= dcrWeights[subfilter][dcrBBox].array
765 newModel = model[dcrBBox].clone()
766 newModel.array += residual
768 badPixels = ~np.isfinite(newModel.array)
769 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
770 if self.config.regularizeModelIterations > 0:
771 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
772 self.config.regularizeModelIterations,
773 self.config.regularizationWidth)
774 newModelImages.append(newModel)
775 if self.config.regularizeModelFrequency > 0:
776 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
777 self.config.regularizeModelFrequency,
778 self.config.regularizationWidth)
779 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
781 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
782 dcrModels.mask, dcrModels.variance)
785 """Calculate a quality of fit metric for the matched templates. 789 dcrModels : `lsst.pipe.tasks.DcrModel` 790 Best fit model of the true sky after correcting chromatic effects. 791 subExposures : `dict` of `lsst.afw.image.ExposureF` 792 The pre-loaded exposures for the current subregion. 793 bbox : `lsst.geom.box.Box2I` 795 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 796 The data references to the input warped exposures. 797 weightList : `list` of `float` 798 The weight to give each input exposure in the coadd 799 statsCtrl : `lsst.afw.math.StatisticsControl` 800 Statistics control object for coadd 804 convergenceMetric : `float` 805 Quality of fit metric for all input exposures, within the sub-region 807 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
809 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
811 if np.max(significanceImage) == 0:
812 significanceImage += 1.
816 for warpExpRef, expWeight
in zip(warpRefList, weightList):
817 exposure = subExposures[warpExpRef.dataId[
"visit"]][bbox]
819 metric += singleMetric
820 metricList[warpExpRef.dataId[
"visit"]] = singleMetric
822 self.log.info(
"Individual metrics:\n%s", metricList)
823 return 1.0
if weight == 0.0
else metric/weight
826 """Calculate a quality of fit metric for a single matched template. 830 dcrModels : `lsst.pipe.tasks.DcrModel` 831 Best fit model of the true sky after correcting chromatic effects. 832 exposure : `lsst.afw.image.ExposureF` 833 The input warped exposure to evaluate. 834 significanceImage : `numpy.ndarray` 835 Array of weights for each pixel corresponding to its significance 836 for the convergence calculation. 837 statsCtrl : `lsst.afw.math.StatisticsControl` 838 Statistics control object for coadd 842 convergenceMetric : `float` 843 Quality of fit metric for one exposure, within the sub-region. 845 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
846 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
847 order=self.config.imageInterpOrder,
848 splitSubfilters=self.config.splitSubfilters,
849 splitThreshold=self.config.splitThreshold,
850 amplifyModel=self.config.accelerateModel)
851 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
852 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
854 finitePixels = np.isfinite(diffVals)
855 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
856 convergeMaskPixels = exposure.mask.array & convergeMask > 0
857 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
858 if np.sum(usePixels) == 0:
861 diffUse = diffVals[usePixels]
862 refUse = refVals[usePixels]
863 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
867 """Add a list of sub-band coadds together. 871 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 872 A list of coadd exposures, each exposure containing 873 the model for one subfilter. 877 coaddExposure : `lsst.afw.image.ExposureF` 878 A single coadd exposure that is the sum of the sub-bands. 880 coaddExposure = dcrCoadds[0].clone()
881 for coadd
in dcrCoadds[1:]:
882 coaddExposure.maskedImage += coadd.maskedImage
885 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
886 mask=None, variance=None):
887 """Create a list of coadd exposures from a list of masked images. 891 dcrModels : `lsst.pipe.tasks.DcrModel` 892 Best fit model of the true sky after correcting chromatic effects. 893 skyInfo : `lsst.pipe.base.Struct` 894 Patch geometry information, from getSkyInfo 895 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 896 The data references to the input warped exposures. 897 weightList : `list` of `float` 898 The weight to give each input exposure in the coadd 899 calibration : `lsst.afw.Image.PhotoCalib`, optional 900 Scale factor to set the photometric calibration of an exposure. 901 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 902 A record of the observations that are included in the coadd. 903 mask : `lsst.afw.image.Mask`, optional 904 Optional mask to override the values in the final coadd. 905 variance : `lsst.afw.image.Image`, optional 906 Optional variance plane to override the values in the final coadd. 910 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 911 A list of coadd exposures, each exposure containing 912 the model for one subfilter. 915 refModel = dcrModels.getReferenceImage()
916 for model
in dcrModels:
917 if self.config.accelerateModel > 1:
918 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
919 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
920 if calibration
is not None:
921 coaddExposure.setPhotoCalib(calibration)
922 if coaddInputs
is not None:
923 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
927 coaddExposure.setPsf(dcrModels.psf)
928 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
929 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
930 maskedImage.image = model
931 maskedImage.mask = dcrModels.mask
932 maskedImage.variance = dcrModels.variance
933 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
935 coaddExposure.setMask(mask)
936 if variance
is not None:
937 coaddExposure.setVariance(variance)
938 dcrCoadds.append(coaddExposure)
942 """Calculate the gain to use for the current iteration. 944 After calculating a new DcrModel, each value is averaged with the 945 value in the corresponding pixel from the previous iteration. This 946 reduces oscillating solutions that iterative techniques are plagued by, 947 and speeds convergence. By far the biggest changes to the model 948 happen in the first couple iterations, so we can also use a more 949 aggressive gain later when the model is changing slowly. 953 convergenceList : `list` of `float` 954 The quality of fit metric from each previous iteration. 955 gainList : `list` of `float` 956 The gains used in each previous iteration: appended with the new 958 Gains are numbers between ``self.config.baseGain`` and 1. 963 Relative weight to give the new solution when updating the model. 964 A value of 1.0 gives equal weight to both solutions. 969 If ``len(convergenceList) != len(gainList)+1``. 971 nIter = len(convergenceList)
972 if nIter != len(gainList) + 1:
973 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 974 % (len(convergenceList), len(gainList)))
976 if self.config.baseGain
is None:
979 baseGain = 1./(self.config.dcrNumSubfilters - 1)
981 baseGain = self.config.baseGain
983 if self.config.useProgressiveGain
and nIter > 2:
991 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
992 for i
in range(nIter - 1)]
995 estFinalConv = np.array(estFinalConv)
996 estFinalConv[estFinalConv < 0] = 0
998 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
999 lastGain = gainList[-1]
1000 lastConv = convergenceList[-2]
1001 newConv = convergenceList[-1]
1006 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1012 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1013 newGain = 1 - abs(delta)
1015 newGain = (newGain + lastGain)/2.
1016 gain = max(baseGain, newGain)
1019 gainList.append(gain)
1023 """Build an array that smoothly tapers to 0 away from detected sources. 1027 dcrModels : `lsst.pipe.tasks.DcrModel` 1028 Best fit model of the true sky after correcting chromatic effects. 1029 dcrBBox : `lsst.geom.box.Box2I` 1030 Sub-region of the coadd which includes a buffer to allow for DCR. 1034 weights : `numpy.ndarray` or `float` 1035 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1036 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1041 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 1043 if not self.config.useModelWeights:
1045 if self.config.modelWeightsWidth < 0:
1046 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1047 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1048 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1049 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1050 weights[convergeMaskPixels] = 1.
1051 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1052 weights /= np.max(weights)
1056 """Smoothly replace model pixel values with those from a 1057 reference at locations away from detected sources. 1061 modelImages : `list` of `lsst.afw.image.Image` 1062 The new DCR model images from the current iteration. 1063 The values will be modified in place. 1064 refImage : `lsst.afw.image.MaskedImage` 1065 A reference image used to supply the default pixel values. 1066 modelWeights : `numpy.ndarray` or `float` 1067 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1068 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1070 if self.config.useModelWeights:
1071 for model
in modelImages:
1072 model.array *= modelWeights
1073 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1076 """Pre-load sub-regions of a list of exposures. 1080 bbox : `lsst.geom.box.Box2I` 1082 statsCtrl : `lsst.afw.math.StatisticsControl` 1083 Statistics control object for coadd 1084 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1085 The data references to the input warped exposures. 1086 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 1087 The image scalars correct for the zero point of the exposures. 1088 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 1089 Each element is dict with keys = mask plane name to add the spans to 1093 subExposures : `dict` 1094 The `dict` keys are the visit IDs, 1095 and the values are `lsst.afw.image.ExposureF` 1096 The pre-loaded exposures for the current subregion. 1097 The variance plane contains weights, and not the variance 1100 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1102 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1103 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
1104 if altMaskSpans
is not None:
1106 imageScaler.scaleMaskedImage(exposure.maskedImage)
1108 exposure.variance.array[:, :] = 0.
1110 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1113 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1114 subExposures[warpExpRef.dataId[
"visit"]] = exposure
1118 """Compute the PSF of the coadd from the exposures with the best seeing. 1122 templateCoadd : `lsst.afw.image.ExposureF` 1123 The initial coadd exposure before accounting for DCR. 1124 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1125 The data references to the input warped exposures. 1129 psf : `lsst.meas.algorithms.CoaddPsf` 1130 The average PSF of the input exposures with the best seeing. 1132 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1136 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1137 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1138 psfSizes = np.zeros(len(ccds))
1139 ccdVisits = np.array(ccds[
"visit"])
1140 for warpExpRef
in warpRefList:
1141 psf = warpExpRef.get(tempExpName).getPsf()
1142 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1143 visit = warpExpRef.dataId[
"visit"]
1144 psfSizes[ccdVisits == visit] = psfSize
1148 sizeThreshold = min(np.median(psfSizes), psfRefSize)
1149 goodPsfs = psfSizes <= sizeThreshold
1150 psf = measAlg.CoaddPsf(ccds[goodPsfs], templateCoadd.getWcs(),
1151 self.config.coaddPsf.makeControl())
def __init__(self, args, kwargs)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def processResults(self, coaddExposure, dataRef)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def _subBBoxIter(bbox, subregionSize)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)