25 from scipy
import ndimage
35 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf
import MeasurePsfTask
38 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
42 dcrNumSubfilters = pexConfig.Field(
44 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
47 maxNumIter = pexConfig.Field(
50 doc=
"Maximum number of iterations of forward modeling.",
53 minNumIter = pexConfig.Field(
56 doc=
"Minimum number of iterations of forward modeling.",
59 convergenceThreshold = pexConfig.Field(
61 doc=
"Target relative change in convergence between iterations of forward modeling.",
64 useConvergence = pexConfig.Field(
66 doc=
"Use convergence test as a forward modeling end condition?" 67 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
70 baseGain = pexConfig.Field(
73 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 74 "A value of 1.0 gives equal weight to both solutions." 75 "Small values imply slower convergence of the solution, but can " 76 "help prevent overshooting and failures in the fit." 77 "If ``baseGain`` is None, a conservative gain " 78 "will be calculated from the number of subfilters. ",
81 useProgressiveGain = pexConfig.Field(
83 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 84 "When calculating the next gain, we use up to 5 previous gains and convergence values." 85 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
88 doAirmassWeight = pexConfig.Field(
90 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
93 modelWeightsWidth = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 useModelWeights = pexConfig.Field(
100 doc=
"Width of the region around detected sources to include in the DcrModel.",
103 splitSubfilters = pexConfig.Field(
105 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 106 "Instead of at the midpoint",
109 splitThreshold = pexConfig.Field(
111 doc=
"Minimum DCR difference within a subfilter to use ``splitSubfilters``, in pixels." 112 "Set to 0 to always split the subfilters.",
115 regularizeModelIterations = pexConfig.Field(
117 doc=
"Maximum relative change of the model allowed between iterations." 118 "Set to zero to disable.",
121 regularizeModelFrequency = pexConfig.Field(
123 doc=
"Maximum relative change of the model allowed between subfilters." 124 "Set to zero to disable.",
127 convergenceMaskPlanes = pexConfig.ListField(
129 default=[
"DETECTED"],
130 doc=
"Mask planes to use to calculate convergence." 132 regularizationWidth = pexConfig.Field(
135 doc=
"Minimum radius of a region to include in regularization, in pixels." 137 imageInterpOrder = pexConfig.Field(
139 doc=
"The order of the spline interpolation used to shift the image plane.",
142 accelerateModel = pexConfig.Field(
144 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
147 doCalculatePsf = pexConfig.Field(
149 doc=
"Set to detect stars and recalculate the PSF from the final coadd." 150 "Otherwise the PSF is estimated from a selection of the best input exposures",
153 detectPsfSources = pexConfig.ConfigurableField(
154 target=measAlg.SourceDetectionTask,
155 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
157 measurePsfSources = pexConfig.ConfigurableField(
158 target=SingleFrameMeasurementTask,
159 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 161 measurePsf = pexConfig.ConfigurableField(
162 target=MeasurePsfTask,
163 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
167 CompareWarpAssembleCoaddConfig.setDefaults(self)
186 self.
measurePsf.starSelector[
"objectSize"].doFluxLimit =
False 190 """Assemble DCR coadded images from a set of warps. 195 The number of pixels to grow each subregion by to allow for DCR. 199 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 200 Warps (also called coadded temporary exposures), including the effects of 201 Differential Chromatic Refraction (DCR). 202 For full details of the mathematics and algorithm, please see 203 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 205 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 206 each subfilter used in the iterative calculation. 207 It begins by dividing the bandpass-defining filter into N equal bandwidth 208 "subfilters", and divides the flux in each pixel from an initial coadd 209 equally into each as a "dcrModel". Because the airmass and parallactic 210 angle of each individual exposure is known, we can calculate the shift 211 relative to the center of the band in each subfilter due to DCR. For each 212 exposure we apply this shift as a linear transformation to the dcrModels 213 and stack the results to produce a DCR-matched exposure. The matched 214 exposures are subtracted from the input exposures to produce a set of 215 residual images, and these residuals are reverse shifted for each 216 exposures' subfilters and stacked. The shifted and stacked residuals are 217 added to the dcrModels to produce a new estimate of the flux in each pixel 218 within each subfilter. The dcrModels are solved for iteratively, which 219 continues until the solution from a new iteration improves by less than 220 a set percentage, or a maximum number of iterations is reached. 221 Two forms of regularization are employed to reduce unphysical results. 222 First, the new solution is averaged with the solution from the previous 223 iteration, which mitigates oscillating solutions where the model 224 overshoots with alternating very high and low values. 225 Second, a common degeneracy when the data have a limited range of airmass or 226 parallactic angle values is for one subfilter to be fit with very low or 227 negative values, while another subfilter is fit with very high values. This 228 typically appears in the form of holes next to sources in one subfilter, 229 and corresponding extended wings in another. Because each subfilter has 230 a narrow bandwidth we assume that physical sources that are above the noise 231 level will not vary in flux by more than a factor of `frequencyClampFactor` 232 between subfilters, and pixels that have flux deviations larger than that 233 factor will have the excess flux distributed evenly among all subfilters. 234 If `splitSubfilters` is set, then each subfilter will be further sub- 235 divided during the forward modeling step (only). This approximates using 236 a higher number of subfilters that may be necessary for high airmass 237 observations, but does not increase the number of free parameters in the 238 fit. This is needed when there are high airmass observations which would 239 otherwise have significant DCR even within a subfilter. Because calculating 240 the shifted images takes most of the time, splitting the subfilters is 241 turned off by way of the `splitThreshold` option for low-airmass 242 observations that do not suffer from DCR within a subfilter. 245 ConfigClass = DcrAssembleCoaddConfig
246 _DefaultName =
"dcrAssembleCoadd" 250 if self.config.doCalculatePsf:
251 self.
schema = afwTable.SourceTable.makeMinimalSchema()
252 self.makeSubtask(
"detectPsfSources", schema=self.
schema)
253 self.makeSubtask(
"measurePsfSources", schema=self.
schema)
254 self.makeSubtask(
"measurePsf", schema=self.
schema)
257 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
258 """Assemble a coadd from a set of warps. 260 Coadd a set of Warps. Compute weights to be applied to each Warp and 261 find scalings to match the photometric zeropoint to a reference Warp. 262 Assemble the Warps using run method. 263 Forward model chromatic effects across multiple subfilters, 264 and subtract from the input Warps to build sets of residuals. 265 Use the residuals to construct a new ``DcrModel`` for each subfilter, 266 and iterate until the model converges. 267 Interpolate over NaNs and optionally write the coadd to disk. 268 Return the coadded exposure. 272 dataRef : `lsst.daf.persistence.ButlerDataRef` 273 Data reference defining the patch for coaddition and the 275 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 276 List of data references to warps. Data to be coadded will be 277 selected from this list based on overlap with the patch defined by 282 results : `lsst.pipe.base.Struct` 283 The Struct contains the following fields: 285 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 286 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 287 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 288 - ``dcrNImages``: `list` of exposure count images for each subfilter 290 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
291 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
293 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
294 warpRefList=warpRefList)
297 self.log.warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
298 skyInfo.patchInfo.getIndex())
300 for subfilter
in range(self.config.dcrNumSubfilters):
302 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
303 AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
304 if self.config.doWrite:
305 self.log.info(
"Persisting dcrCoadd")
306 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
307 numSubfilters=self.config.dcrNumSubfilters)
308 if self.config.doNImage
and results.dcrNImages
is not None:
309 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
310 numSubfilters=self.config.dcrNumSubfilters)
315 """Interpolate over missing data and mask bright stars. 317 Also detect sources on the coadd exposure and measure the final PSF, 318 if ``doCalculatePsf`` is set. 322 coaddExposure : `lsst.afw.image.Exposure` 323 The final coadded exposure. 324 dataRef : `lsst.daf.persistence.ButlerDataRef` 325 Data reference defining the patch for coaddition and the 330 if self.config.doCalculatePsf:
331 expId = dataRef.get(
"dcrCoaddId")
332 table = afwTable.SourceTable.make(self.
schema)
333 detResults = self.detectPsfSources.
run(table, coaddExposure, expId, clearMask=
False)
334 coaddSources = detResults.sources
335 self.measurePsfSources.
run(
336 measCat=coaddSources,
337 exposure=coaddExposure,
341 psfResults = self.measurePsf.
run(coaddExposure, coaddSources, expId=expId)
342 except RuntimeError
as e:
343 self.log.warn(
"Unable to calculate PSF, using default coadd PSF: %s" % e)
345 coaddExposure.setPsf(psfResults.psf)
348 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 350 Sets the property ``bufferSize``. 354 templateCoadd : `lsst.afw.image.ExposureF` 355 The initial coadd exposure before accounting for DCR. 356 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 357 The data references to the input warped exposures. 358 weightList : `list` of `float` 359 The weight to give each input exposure in the coadd 360 Will be modified in place if ``doAirmassWeight`` is set. 364 dcrModels : `lsst.pipe.tasks.DcrModel` 365 Best fit model of the true sky after correcting chromatic effects. 370 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 372 filterInfo = templateCoadd.getFilter()
373 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
374 raise NotImplementedError(
"No minimum/maximum wavelength information found" 375 " in the filter definition! Please add lambdaMin and lambdaMax" 376 " to the Mapper class in your obs package.")
381 for visitNum, warpExpRef
in enumerate(warpRefList):
382 visitInfo = warpExpRef.get(tempExpName +
"_visitInfo")
383 visit = warpExpRef.dataId[
"visit"]
384 airmass = visitInfo.getBoresightAirmass()
385 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
386 airmassDict[visit] = airmass
387 angleDict[visit] = parallacticAngle
388 if self.config.doAirmassWeight:
389 weightList[visitNum] *= airmass
390 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
391 filterInfo, self.config.dcrNumSubfilters))))
392 self.log.info(
"Selected airmasses:\n%s", airmassDict)
393 self.log.info(
"Selected parallactic angles:\n%s", angleDict)
396 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
397 self.config.dcrNumSubfilters,
398 filterInfo=filterInfo,
402 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
403 supplementaryData=None):
404 """Assemble the coadd. 406 Requires additional inputs Struct ``supplementaryData`` to contain a 407 ``templateCoadd`` that serves as the model of the static sky. 409 Find artifacts and apply them to the warps' masks creating a list of 410 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 411 Then pass these alternative masks to the base class's assemble method. 413 Divide the ``templateCoadd`` evenly between each subfilter of a 414 ``DcrModel`` as the starting best estimate of the true wavelength- 415 dependent sky. Forward model the ``DcrModel`` using the known 416 chromatic effects in each subfilter and calculate a convergence metric 417 based on how well the modeled template matches the input warps. If 418 the convergence has not yet reached the desired threshold, then shift 419 and stack the residual images to build a new ``DcrModel``. Apply 420 conditioning to prevent oscillating solutions between iterations or 423 Once the ``DcrModel`` reaches convergence or the maximum number of 424 iterations has been reached, fill the metadata for each subfilter 425 image and make them proper ``coaddExposure``s. 429 skyInfo : `lsst.pipe.base.Struct` 430 Patch geometry information, from getSkyInfo 431 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 432 The data references to the input warped exposures. 433 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 434 The image scalars correct for the zero point of the exposures. 435 weightList : `list` of `float` 436 The weight to give each input exposure in the coadd 437 supplementaryData : `lsst.pipe.base.Struct` 438 Result struct returned by ``makeSupplementaryData`` with components: 440 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 444 result : `lsst.pipe.base.Struct` 445 Result struct with components: 447 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 448 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 449 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 450 - ``dcrNImages``: `list` of exposure count images for each subfilter 452 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
453 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
454 templateCoadd = supplementaryData.templateCoadd
455 baseMask = templateCoadd.mask.clone()
458 baseVariance = templateCoadd.variance.clone()
459 baseVariance /= self.config.dcrNumSubfilters
460 spanSetMaskList = self.
findArtifacts(templateCoadd, warpRefList, imageScalerList)
462 templateCoadd.setMask(baseMask)
463 badMaskPlanes = self.config.badMaskPlanes[:]
468 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
472 if self.config.doNImage:
473 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
474 spanSetMaskList, stats.ctrl)
475 nImage = afwImage.ImageU(skyInfo.bbox)
479 for dcrNImage
in dcrNImages:
484 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
485 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
486 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
488 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
491 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
492 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
493 dcrBBox = afwGeom.Box2I(subBBox)
495 dcrBBox.clip(dcrModels.bbox)
498 imageScalerList, spanSetMaskList)
500 warpRefList, weightList, stats.ctrl)
501 self.log.info(
"Initial convergence : %s", convergenceMetric)
502 convergenceList = [convergenceMetric]
504 convergenceCheck = 1.
505 refImage = templateCoadd.image
506 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
509 stats.ctrl, convergenceMetric, gain,
510 modelWeights, refImage, dcrWeights)
511 if self.config.useConvergence:
513 warpRefList, weightList, stats.ctrl)
514 if convergenceMetric == 0:
515 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 516 "most likely due to there being no valid data in the region.",
517 skyInfo.patchInfo.getIndex(), subIter)
519 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
520 if (convergenceCheck < 0) & (modelIter > minNumIter):
521 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 522 "iterations or desired convergence improvement of %s." 524 skyInfo.patchInfo.getIndex(), subIter,
525 self.config.convergenceThreshold, convergenceCheck)
527 convergenceList.append(convergenceMetric)
528 if modelIter > maxNumIter:
529 if self.config.useConvergence:
530 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 531 "before reaching desired convergence improvement of %s." 532 " Final convergence improvement: %s",
533 skyInfo.patchInfo.getIndex(), subIter,
534 self.config.convergenceThreshold, convergenceCheck)
537 if self.config.useConvergence:
538 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
539 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
542 if self.config.useConvergence:
543 self.log.info(
"Coadd patch %s subregion %s finished with " 544 "convergence metric %s after %s iterations",
545 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
547 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
548 skyInfo.patchInfo.getIndex(), subIter, modelIter)
549 if self.config.useConvergence
and convergenceMetric > 0:
550 self.log.info(
"Final convergence improvement was %.4f%% overall",
551 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
553 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
554 calibration=self.scaleZeroPoint.getPhotoCalib(),
555 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
557 variance=baseVariance)
559 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
560 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
563 """Calculate the number of exposures contributing to each subfilter. 567 dcrModels : `lsst.pipe.tasks.DcrModel` 568 Best fit model of the true sky after correcting chromatic effects. 569 bbox : `lsst.afw.geom.box.Box2I` 570 Bounding box of the patch to coadd. 571 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 572 The data references to the input warped exposures. 573 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 574 Each element of the `dict` contains the new mask plane name 575 (e.g. "CLIPPED and/or "NO_DATA") as the key, 576 and the list of SpanSets to apply to the mask. 577 statsCtrl : `lsst.afw.math.StatisticsControl` 578 Statistics control object for coadd 582 dcrNImages : `list` of `lsst.afw.image.ImageU` 583 List of exposure count images for each subfilter 584 dcrWeights : `list` of `lsst.afw.image.ImageF` 585 Per-pixel weights for each subfilter. 586 Equal to 1/(number of unmasked images contributing to each pixel). 588 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
589 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
591 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
592 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
593 visitInfo = exposure.getInfo().getVisitInfo()
594 wcs = exposure.getInfo().getWcs()
596 if altMaskSpans
is not None:
598 weightImage = np.zeros_like(exposure.image.array)
599 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
602 weightsGenerator = self.
dcrResiduals(weightImage, visitInfo, wcs, dcrModels.filter)
603 for shiftedWeights, dcrNImage, dcrWeight
in zip(weightsGenerator, dcrNImages, dcrWeights):
604 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
605 dcrWeight.array += shiftedWeights
607 weightsThreshold = 1.
608 goodPix = dcrWeights[0].array > weightsThreshold
609 for weights
in dcrWeights[1:]:
610 goodPix = (weights.array > weightsThreshold) & goodPix
611 for subfilter
in range(self.config.dcrNumSubfilters):
612 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
613 dcrWeights[subfilter].array[~goodPix] = 0.
614 dcrNImages[subfilter].array[~goodPix] = 0
615 return (dcrNImages, dcrWeights)
618 statsCtrl, convergenceMetric,
619 gain, modelWeights, refImage, dcrWeights):
620 """Assemble the DCR coadd for a sub-region. 622 Build a DCR-matched template for each input exposure, then shift the 623 residuals according to the DCR in each subfilter. 624 Stack the shifted residuals and apply them as a correction to the 625 solution from the previous iteration. 626 Restrict the new model solutions from varying by more than a factor of 627 `modelClampFactor` from the last solution, and additionally restrict the 628 individual subfilter models from varying by more than a factor of 629 `frequencyClampFactor` from their average. 630 Finally, mitigate potentially oscillating solutions by averaging the new 631 solution with the solution from the previous iteration, weighted by 632 their convergence metric. 636 dcrModels : `lsst.pipe.tasks.DcrModel` 637 Best fit model of the true sky after correcting chromatic effects. 638 subExposures : `dict` of `lsst.afw.image.ExposureF` 639 The pre-loaded exposures for the current subregion. 640 bbox : `lsst.afw.geom.box.Box2I` 641 Bounding box of the subregion to coadd. 642 dcrBBox : `lsst.afw.geom.box.Box2I` 643 Sub-region of the coadd which includes a buffer to allow for DCR. 644 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 645 The data references to the input warped exposures. 646 statsCtrl : `lsst.afw.math.StatisticsControl` 647 Statistics control object for coadd 648 convergenceMetric : `float` 649 Quality of fit metric for the matched templates of the input images. 650 gain : `float`, optional 651 Relative weight to give the new solution when updating the model. 652 modelWeights : `numpy.ndarray` or `float` 653 A 2D array of weight values that tapers smoothly to zero away from detected sources. 654 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 655 refImage : `lsst.afw.image.Image` 656 A reference image used to supply the default pixel values. 657 dcrWeights : `list` of `lsst.afw.image.Image` 658 Per-pixel weights for each subfilter. 659 Equal to 1/(number of unmasked images contributing to each pixel). 661 residualGeneratorList = []
663 for warpExpRef
in warpRefList:
664 exposure = subExposures[warpExpRef.dataId[
"visit"]]
665 visitInfo = exposure.getInfo().getVisitInfo()
666 wcs = exposure.getInfo().getWcs()
667 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
668 order=self.config.imageInterpOrder,
669 splitSubfilters=self.config.splitSubfilters,
670 splitThreshold=self.config.splitThreshold,
671 amplifyModel=self.config.accelerateModel)
672 residual = exposure.image.array - templateImage.array
674 residual *= exposure.variance.array
678 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
680 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
682 modelWeights=modelWeights,
684 dcrWeights=dcrWeights)
685 dcrModels.assign(dcrSubModelOut, bbox)
688 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 692 residual : `numpy.ndarray` 693 The residual masked image for one exposure, 694 after subtracting the matched template 695 visitInfo : `lsst.afw.image.VisitInfo` 696 Metadata for the exposure. 697 wcs : `lsst.afw.geom.SkyWcs` 698 Coordinate system definition (wcs) for the exposure. 699 filterInfo : `lsst.afw.image.Filter` 700 The filter definition, set in the current instruments' obs package. 701 Required for any calculation of DCR, including making matched templates. 705 residualImage : `numpy.ndarray` 706 The residual image for the next subfilter, shifted for DCR. 710 filteredResidual = ndimage.spline_filter(residual, order=self.config.imageInterpOrder)
713 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters,
714 splitSubfilters=
False)
716 yield applyDcr(filteredResidual, dcr, useInverse=
True, splitSubfilters=
False,
717 doPrefilter=
False, order=self.config.imageInterpOrder)
720 gain, modelWeights, refImage, dcrWeights):
721 """Calculate a new DcrModel from a set of image residuals. 725 dcrModels : `lsst.pipe.tasks.DcrModel` 726 Current model of the true sky after correcting chromatic effects. 727 residualGeneratorList : `generator` of `numpy.ndarray` 728 The residual image for the next subfilter, shifted for DCR. 729 dcrBBox : `lsst.afw.geom.box.Box2I` 730 Sub-region of the coadd which includes a buffer to allow for DCR. 731 statsCtrl : `lsst.afw.math.StatisticsControl` 732 Statistics control object for coadd 734 Relative weight to give the new solution when updating the model. 735 modelWeights : `numpy.ndarray` or `float` 736 A 2D array of weight values that tapers smoothly to zero away from detected sources. 737 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 738 refImage : `lsst.afw.image.Image` 739 A reference image used to supply the default pixel values. 740 dcrWeights : `list` of `lsst.afw.image.Image` 741 Per-pixel weights for each subfilter. 742 Equal to 1/(number of unmasked images contributing to each pixel). 746 dcrModel : `lsst.pipe.tasks.DcrModel` 747 New model of the true sky after correcting chromatic effects. 750 for subfilter, model
in enumerate(dcrModels):
751 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
752 residual = np.sum(residualsList, axis=0)
753 residual *= dcrWeights[subfilter][dcrBBox].array
755 newModel = model[dcrBBox].clone()
756 newModel.array += residual
758 badPixels = ~np.isfinite(newModel.array)
759 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
760 if self.config.regularizeModelIterations > 0:
761 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
762 self.config.regularizeModelIterations,
763 self.config.regularizationWidth)
764 newModelImages.append(newModel)
765 if self.config.regularizeModelFrequency > 0:
766 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
767 self.config.regularizeModelFrequency,
768 self.config.regularizationWidth)
769 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
771 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
772 dcrModels.mask, dcrModels.variance)
775 """Calculate a quality of fit metric for the matched templates. 779 dcrModels : `lsst.pipe.tasks.DcrModel` 780 Best fit model of the true sky after correcting chromatic effects. 781 subExposures : `dict` of `lsst.afw.image.ExposureF` 782 The pre-loaded exposures for the current subregion. 783 bbox : `lsst.afw.geom.box.Box2I` 785 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 786 The data references to the input warped exposures. 787 weightList : `list` of `float` 788 The weight to give each input exposure in the coadd 789 statsCtrl : `lsst.afw.math.StatisticsControl` 790 Statistics control object for coadd 794 convergenceMetric : `float` 795 Quality of fit metric for all input exposures, within the sub-region 797 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
799 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
801 if np.max(significanceImage) == 0:
802 significanceImage += 1.
806 for warpExpRef, expWeight
in zip(warpRefList, weightList):
807 exposure = subExposures[warpExpRef.dataId[
"visit"]][bbox]
809 metric += singleMetric
810 metricList[warpExpRef.dataId[
"visit"]] = singleMetric
812 self.log.info(
"Individual metrics:\n%s", metricList)
813 return 1.0
if weight == 0.0
else metric/weight
816 """Calculate a quality of fit metric for a single matched template. 820 dcrModels : `lsst.pipe.tasks.DcrModel` 821 Best fit model of the true sky after correcting chromatic effects. 822 exposure : `lsst.afw.image.ExposureF` 823 The input warped exposure to evaluate. 824 significanceImage : `numpy.ndarray` 825 Array of weights for each pixel corresponding to its significance 826 for the convergence calculation. 827 statsCtrl : `lsst.afw.math.StatisticsControl` 828 Statistics control object for coadd 832 convergenceMetric : `float` 833 Quality of fit metric for one exposure, within the sub-region. 835 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
836 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
837 order=self.config.imageInterpOrder,
838 splitSubfilters=self.config.splitSubfilters,
839 splitThreshold=self.config.splitThreshold,
840 amplifyModel=self.config.accelerateModel)
841 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
842 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
844 finitePixels = np.isfinite(diffVals)
845 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
846 convergeMaskPixels = exposure.mask.array & convergeMask > 0
847 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
848 if np.sum(usePixels) == 0:
851 diffUse = diffVals[usePixels]
852 refUse = refVals[usePixels]
853 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
857 """Add a list of sub-band coadds together. 861 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 862 A list of coadd exposures, each exposure containing 863 the model for one subfilter. 867 coaddExposure : `lsst.afw.image.ExposureF` 868 A single coadd exposure that is the sum of the sub-bands. 870 coaddExposure = dcrCoadds[0].clone()
871 for coadd
in dcrCoadds[1:]:
872 coaddExposure.maskedImage += coadd.maskedImage
875 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
876 mask=None, variance=None):
877 """Create a list of coadd exposures from a list of masked images. 881 dcrModels : `lsst.pipe.tasks.DcrModel` 882 Best fit model of the true sky after correcting chromatic effects. 883 skyInfo : `lsst.pipe.base.Struct` 884 Patch geometry information, from getSkyInfo 885 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 886 The data references to the input warped exposures. 887 weightList : `list` of `float` 888 The weight to give each input exposure in the coadd 889 calibration : `lsst.afw.Image.PhotoCalib`, optional 890 Scale factor to set the photometric calibration of an exposure. 891 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 892 A record of the observations that are included in the coadd. 893 mask : `lsst.afw.image.Mask`, optional 894 Optional mask to override the values in the final coadd. 895 variance : `lsst.afw.image.Image`, optional 896 Optional variance plane to override the values in the final coadd. 900 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 901 A list of coadd exposures, each exposure containing 902 the model for one subfilter. 905 refModel = dcrModels.getReferenceImage()
906 for model
in dcrModels:
907 if self.config.accelerateModel > 1:
908 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
909 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
910 if calibration
is not None:
911 coaddExposure.setPhotoCalib(calibration)
912 if coaddInputs
is not None:
913 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
917 coaddExposure.setPsf(dcrModels.psf)
918 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
919 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
920 maskedImage.image = model
921 maskedImage.mask = dcrModels.mask
922 maskedImage.variance = dcrModels.variance
923 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
925 coaddExposure.setMask(mask)
926 if variance
is not None:
927 coaddExposure.setVariance(variance)
928 dcrCoadds.append(coaddExposure)
932 """Calculate the gain to use for the current iteration. 934 After calculating a new DcrModel, each value is averaged with the 935 value in the corresponding pixel from the previous iteration. This 936 reduces oscillating solutions that iterative techniques are plagued by, 937 and speeds convergence. By far the biggest changes to the model 938 happen in the first couple iterations, so we can also use a more 939 aggressive gain later when the model is changing slowly. 943 convergenceList : `list` of `float` 944 The quality of fit metric from each previous iteration. 945 gainList : `list` of `float` 946 The gains used in each previous iteration: appended with the new 948 Gains are numbers between ``self.config.baseGain`` and 1. 953 Relative weight to give the new solution when updating the model. 954 A value of 1.0 gives equal weight to both solutions. 959 If ``len(convergenceList) != len(gainList)+1``. 961 nIter = len(convergenceList)
962 if nIter != len(gainList) + 1:
963 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 964 % (len(convergenceList), len(gainList)))
966 if self.config.baseGain
is None:
969 baseGain = 1./(self.config.dcrNumSubfilters - 1)
971 baseGain = self.config.baseGain
973 if self.config.useProgressiveGain
and nIter > 2:
981 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
982 for i
in range(nIter - 1)]
985 estFinalConv = np.array(estFinalConv)
986 estFinalConv[estFinalConv < 0] = 0
988 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
989 lastGain = gainList[-1]
990 lastConv = convergenceList[-2]
991 newConv = convergenceList[-1]
996 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
1002 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
1003 newGain = 1 - abs(delta)
1005 newGain = (newGain + lastGain)/2.
1006 gain = max(baseGain, newGain)
1009 gainList.append(gain)
1013 """Build an array that smoothly tapers to 0 away from detected sources. 1017 dcrModels : `lsst.pipe.tasks.DcrModel` 1018 Best fit model of the true sky after correcting chromatic effects. 1019 dcrBBox : `lsst.afw.geom.box.Box2I` 1020 Sub-region of the coadd which includes a buffer to allow for DCR. 1024 weights : `numpy.ndarray` or `float` 1025 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1026 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1031 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 1033 if not self.config.useModelWeights:
1035 if self.config.modelWeightsWidth < 0:
1036 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1037 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1038 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1039 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1040 weights[convergeMaskPixels] = 1.
1041 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1042 weights /= np.max(weights)
1046 """Smoothly replace model pixel values with those from a 1047 reference at locations away from detected sources. 1051 modelImages : `list` of `lsst.afw.image.Image` 1052 The new DCR model images from the current iteration. 1053 The values will be modified in place. 1054 refImage : `lsst.afw.image.MaskedImage` 1055 A reference image used to supply the default pixel values. 1056 modelWeights : `numpy.ndarray` or `float` 1057 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1058 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1060 if self.config.useModelWeights:
1061 for model
in modelImages:
1062 model.array *= modelWeights
1063 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1066 """Pre-load sub-regions of a list of exposures. 1070 bbox : `lsst.afw.geom.box.Box2I` 1072 statsCtrl : `lsst.afw.math.StatisticsControl` 1073 Statistics control object for coadd 1074 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1075 The data references to the input warped exposures. 1076 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 1077 The image scalars correct for the zero point of the exposures. 1078 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 1079 Each element is dict with keys = mask plane name to add the spans to 1083 subExposures : `dict` 1084 The `dict` keys are the visit IDs, 1085 and the values are `lsst.afw.image.ExposureF` 1086 The pre-loaded exposures for the current subregion. 1087 The variance plane contains weights, and not the variance 1090 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1092 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1093 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
1094 if altMaskSpans
is not None:
1096 imageScaler.scaleMaskedImage(exposure.maskedImage)
1098 exposure.variance.array[:, :] = 0.
1100 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1103 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1104 subExposures[warpExpRef.dataId[
"visit"]] = exposure
1108 """Compute the PSF of the coadd from the exposures with the best seeing. 1112 templateCoadd : `lsst.afw.image.ExposureF` 1113 The initial coadd exposure before accounting for DCR. 1114 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1115 The data references to the input warped exposures. 1119 psf : `lsst.meas.algorithms.CoaddPsf` 1120 The average PSF of the input exposures with the best seeing. 1122 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1124 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1125 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1127 for visitNum, warpExpRef
in enumerate(warpRefList):
1128 psf = warpExpRef.get(tempExpName).getPsf()
1129 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1130 psfSizeList.append(psfSize)
1134 sizeThreshold = min(np.median(psfSizeList), psfRefSize)
1135 goodVisits = np.array(psfSizeList) <= sizeThreshold
1136 psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1137 self.config.coaddPsf.makeControl())
def __init__(self, args, kwargs)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def processResults(self, coaddExposure, dataRef)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def _subBBoxIter(bbox, subregionSize)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)