25 from scipy
import ndimage
33 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
35 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
39 dcrNumSubfilters = pexConfig.Field(
41 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
44 maxNumIter = pexConfig.Field(
46 doc=
"Maximum number of iterations of forward modeling.",
49 minNumIter = pexConfig.Field(
51 doc=
"Minimum number of iterations of forward modeling.",
54 convergenceThreshold = pexConfig.Field(
56 doc=
"Target relative change in convergence between iterations of forward modeling.",
59 useConvergence = pexConfig.Field(
61 doc=
"Use convergence test as a forward modeling end condition?" 62 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
65 baseGain = pexConfig.Field(
68 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 69 "A value of 1.0 gives equal weight to both solutions." 70 "Small values imply slower convergence of the solution, but can " 71 "help prevent overshooting and failures in the fit." 72 "If ``baseGain`` is None, a conservative gain " 73 "will be calculated from the number of subfilters. ",
76 useProgressiveGain = pexConfig.Field(
78 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 79 "When calculating the next gain, we use up to 5 previous gains and convergence values." 80 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
83 doAirmassWeight = pexConfig.Field(
85 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
88 modelWeightsWidth = pexConfig.Field(
90 doc=
"Width of the region around detected sources to include in the DcrModel.",
93 useModelWeights = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 splitSubfilters = pexConfig.Field(
100 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 101 "Instead of at the midpoint",
104 regularizeModelIterations = pexConfig.Field(
106 doc=
"Maximum relative change of the model allowed between iterations." 107 "Set to zero to disable.",
110 regularizeModelFrequency = pexConfig.Field(
112 doc=
"Maximum relative change of the model allowed between subfilters." 113 "Set to zero to disable.",
116 convergenceMaskPlanes = pexConfig.ListField(
118 default=[
"DETECTED"],
119 doc=
"Mask planes to use to calculate convergence." 121 regularizationWidth = pexConfig.Field(
124 doc=
"Minimum radius of a region to include in regularization, in pixels." 126 imageWarpMethod = pexConfig.Field(
128 doc=
"Name of the warping kernel to use for shifting the image and variance planes.",
131 maskWarpMethod = pexConfig.Field(
133 doc=
"Name of the warping kernel to use for shifting the mask plane.",
138 CompareWarpAssembleCoaddConfig.setDefaults(self)
149 """Assemble DCR coadded images from a set of warps. 154 The number of pixels to grow each subregion by to allow for DCR. 155 warpCtrl : `lsst.afw.math.WarpingControl` 156 Configuration settings for warping an image 160 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 161 Warps (also called coadded temporary exposures), including the effects of 162 Differential Chromatic Refraction (DCR). 163 For full details of the mathematics and algorithm, please see 164 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 166 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 167 each subfilter used in the iterative calculation. 168 It begins by dividing the bandpass-defining filter into N equal bandwidth 169 "subfilters", and divides the flux in each pixel from an initial coadd 170 equally into each as a "dcrModel". Because the airmass and parallactic 171 angle of each individual exposure is known, we can calculate the shift 172 relative to the center of the band in each subfilter due to DCR. For each 173 exposure we apply this shift as a linear transformation to the dcrModels 174 and stack the results to produce a DCR-matched exposure. The matched 175 exposures are subtracted from the input exposures to produce a set of 176 residual images, and these residuals are reverse shifted for each 177 exposures' subfilters and stacked. The shifted and stacked residuals are 178 added to the dcrModels to produce a new estimate of the flux in each pixel 179 within each subfilter. The dcrModels are solved for iteratively, which 180 continues until the solution from a new iteration improves by less than 181 a set percentage, or a maximum number of iterations is reached. 182 Two forms of regularization are employed to reduce unphysical results. 183 First, the new solution is averaged with the solution from the previous 184 iteration, which mitigates oscillating solutions where the model 185 overshoots with alternating very high and low values. 186 Second, a common degeneracy when the data have a limited range of airmass or 187 parallactic angle values is for one subfilter to be fit with very low or 188 negative values, while another subfilter is fit with very high values. This 189 typically appears in the form of holes next to sources in one subfilter, 190 and corresponding extended wings in another. Because each subfilter has 191 a narrow bandwidth we assume that physical sources that are above the noise 192 level will not vary in flux by more than a factor of `frequencyClampFactor` 193 between subfilters, and pixels that have flux deviations larger than that 194 factor will have the excess flux distributed evenly among all subfilters. 197 ConfigClass = DcrAssembleCoaddConfig
198 _DefaultName =
"dcrAssembleCoadd" 202 """Assemble a coadd from a set of warps. 204 Coadd a set of Warps. Compute weights to be applied to each Warp and 205 find scalings to match the photometric zeropoint to a reference Warp. 206 Assemble the Warps using run method. 207 Forward model chromatic effects across multiple subfilters, 208 and subtract from the input Warps to build sets of residuals. 209 Use the residuals to construct a new ``DcrModel`` for each subfilter, 210 and iterate until the model converges. 211 Interpolate over NaNs and optionally write the coadd to disk. 212 Return the coadded exposure. 216 dataRef : `lsst.daf.persistence.ButlerDataRef` 217 Data reference defining the patch for coaddition and the 219 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 220 List of data references to warps. Data to be coadded will be 221 selected from this list based on overlap with the patch defined by 226 results : `lsst.pipe.base.Struct` 227 The Struct contains the following fields: 229 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 230 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 231 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 232 - ``dcrNImages``: `list` of exposure count images for each subfilter 234 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
235 for subfilter
in range(self.config.dcrNumSubfilters):
237 if self.config.doWrite:
238 self.log.info(
"Persisting dcrCoadd")
239 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
240 numSubfilters=self.config.dcrNumSubfilters)
241 if self.config.doNImage
and results.dcrNImages
is not None:
242 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
243 numSubfilters=self.config.dcrNumSubfilters)
248 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 250 Sets the properties ``warpCtrl`` and ``bufferSize``. 254 templateCoadd : `lsst.afw.image.ExposureF` 255 The initial coadd exposure before accounting for DCR. 256 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 257 The data references to the input warped exposures. 258 weightList : `list` of `float` 259 The weight to give each input exposure in the coadd 260 Will be modified in place if ``doAirmassWeight`` is set. 264 dcrModels : `lsst.pipe.tasks.DcrModel` 265 Best fit model of the true sky after correcting chromatic effects. 270 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 272 filterInfo = templateCoadd.getFilter()
273 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
274 raise NotImplementedError(
"No minimum/maximum wavelength information found" 275 " in the filter definition! Please add lambdaMin and lambdaMax" 276 " to the Mapper class in your obs package.")
279 for visitNum, tempExpRef
in enumerate(tempExpRefList):
280 visitInfo = tempExpRef.get(tempExpName +
"_visitInfo")
281 airmass = visitInfo.getBoresightAirmass()
282 if self.config.doAirmassWeight:
283 weightList[visitNum] *= airmass
284 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
285 filterInfo, self.config.dcrNumSubfilters))))
290 warpInterpLength = max(self.config.subregionSize)
291 self.
warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
292 self.config.maskWarpMethod,
293 cacheSize=warpCache, interpLength=warpInterpLength)
294 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
295 self.config.dcrNumSubfilters,
296 filterInfo=filterInfo,
297 psf=templateCoadd.getPsf())
300 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
301 supplementaryData=None):
302 """Assemble the coadd. 304 Requires additional inputs Struct ``supplementaryData`` to contain a 305 ``templateCoadd`` that serves as the model of the static sky. 307 Find artifacts and apply them to the warps' masks creating a list of 308 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 309 Then pass these alternative masks to the base class's assemble method. 311 Divide the ``templateCoadd`` evenly between each subfilter of a 312 ``DcrModel`` as the starting best estimate of the true wavelength- 313 dependent sky. Forward model the ``DcrModel`` using the known 314 chromatic effects in each subfilter and calculate a convergence metric 315 based on how well the modeled template matches the input warps. If 316 the convergence has not yet reached the desired threshold, then shift 317 and stack the residual images to build a new ``DcrModel``. Apply 318 conditioning to prevent oscillating solutions between iterations or 321 Once the ``DcrModel`` reaches convergence or the maximum number of 322 iterations has been reached, fill the metadata for each subfilter 323 image and make them proper ``coaddExposure``s. 327 skyInfo : `lsst.pipe.base.Struct` 328 Patch geometry information, from getSkyInfo 329 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 330 The data references to the input warped exposures. 331 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 332 The image scalars correct for the zero point of the exposures. 333 weightList : `list` of `float` 334 The weight to give each input exposure in the coadd 335 supplementaryData : `lsst.pipe.base.Struct` 336 Result struct returned by ``makeSupplementaryData`` with components: 338 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 342 result : `lsst.pipe.base.Struct` 343 Result struct with components: 345 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 346 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 347 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 348 - ``dcrNImages``: `list` of exposure count images for each subfilter 350 templateCoadd = supplementaryData.templateCoadd
351 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
352 badMaskPlanes = self.config.badMaskPlanes[:]
353 badMaskPlanes.append(
"CLIPPED")
354 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
360 dcrModels = self.
prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
361 if self.config.doNImage:
363 tempExpRefList, spanSetMaskList, stats.ctrl)
364 nImage = afwImage.ImageU(skyInfo.bbox)
368 for dcrNImage
in dcrNImages:
373 baseMask = templateCoadd.mask
374 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
375 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
376 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
378 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
381 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
382 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
383 dcrBBox = afwGeom.Box2I(subBBox)
385 dcrBBox.clip(dcrModels.bbox)
386 if self.config.useModelWeights:
391 imageScalerList, weightList, spanSetMaskList,
393 self.log.info(
"Initial convergence : %s", convergenceMetric)
394 convergenceList = [convergenceMetric]
396 convergenceCheck = 1.
397 subfilterVariance =
None 398 while (convergenceCheck > self.config.convergenceThreshold
or 399 modelIter < self.config.minNumIter):
402 weightList, spanSetMaskList, stats.flags, stats.ctrl,
403 convergenceMetric, baseMask, subfilterVariance, gain,
405 if self.config.useConvergence:
407 imageScalerList, weightList,
410 if convergenceMetric == 0:
411 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 412 "most likely due to there being no valid data in the region.",
413 skyInfo.patchInfo.getIndex(), subIter)
415 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
416 if convergenceCheck < 0:
417 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 418 "iterations or desired convergence improvement of %s." 420 skyInfo.patchInfo.getIndex(), subIter,
421 self.config.convergenceThreshold, convergenceCheck)
423 convergenceList.append(convergenceMetric)
424 if modelIter > self.config.maxNumIter:
425 if self.config.useConvergence:
426 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 427 "before reaching desired convergence improvement of %s." 428 " Final convergence improvement: %s",
429 skyInfo.patchInfo.getIndex(), subIter,
430 self.config.convergenceThreshold, convergenceCheck)
433 if self.config.useConvergence:
434 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
435 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
438 if self.config.useConvergence:
439 self.log.info(
"Coadd patch %s subregion %s finished with " 440 "convergence metric %s after %s iterations",
441 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
443 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
444 skyInfo.patchInfo.getIndex(), subIter, modelIter)
445 if self.config.useConvergence
and convergenceMetric > 0:
446 self.log.info(
"Final convergence improvement was %.4f%% overall",
447 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
449 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
450 calibration=self.scaleZeroPoint.getCalib(),
451 coaddInputs=self.inputRecorder.makeCoaddInputs(),
452 mask=templateCoadd.mask)
454 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
455 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
457 def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
458 """Calculate the number of exposures contributing to each subfilter. 462 dcrModels : `lsst.pipe.tasks.DcrModel` 463 Best fit model of the true sky after correcting chromatic effects. 464 bbox : `lsst.afw.geom.box.Box2I` 465 Bounding box of the patch to coadd. 466 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 467 The data references to the input warped exposures. 468 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 469 Each element is dict with keys = mask plane name to add the spans to 470 statsCtrl : `lsst.afw.math.StatisticsControl` 471 Statistics control object for coadd 475 dcrNImages : `list` of `lsst.afw.image.ImageU` 476 List of exposure count images for each subfilter 478 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
480 for tempExpRef, altMaskSpans
in zip(tempExpRefList, spanSetMaskList):
481 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
482 visitInfo = exposure.getInfo().getVisitInfo()
483 wcs = exposure.getInfo().getWcs()
485 if altMaskSpans
is not None:
487 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
488 for dcr, dcrNImage
in zip(dcrShift, dcrNImages):
489 shiftedImage = applyDcr(exposure.maskedImage, dcr, self.
warpCtrl, useInverse=
True)
490 dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
493 def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
494 spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
495 baseMask, subfilterVariance, gain, modelWeights):
496 """Assemble the DCR coadd for a sub-region. 498 Build a DCR-matched template for each input exposure, then shift the 499 residuals according to the DCR in each subfilter. 500 Stack the shifted residuals and apply them as a correction to the 501 solution from the previous iteration. 502 Restrict the new model solutions from varying by more than a factor of 503 `modelClampFactor` from the last solution, and additionally restrict the 504 individual subfilter models from varying by more than a factor of 505 `frequencyClampFactor` from their average. 506 Finally, mitigate potentially oscillating solutions by averaging the new 507 solution with the solution from the previous iteration, weighted by 508 their convergence metric. 512 dcrModels : `lsst.pipe.tasks.DcrModel` 513 Best fit model of the true sky after correcting chromatic effects. 514 bbox : `lsst.afw.geom.box.Box2I` 515 Bounding box of the subregion to coadd. 516 dcrBBox :`lsst.afw.geom.box.Box2I` 517 Sub-region of the coadd which includes a buffer to allow for DCR. 518 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 519 The data references to the input warped exposures. 520 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 521 The image scalars correct for the zero point of the exposures. 522 weightList : `list` of `float` 523 The weight to give each input exposure in the coadd 524 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 525 Each element is dict with keys = mask plane name to add the spans to 526 statsFlags : `lsst.afw.math.Property` 527 Statistics settings for coaddition. 528 statsCtrl : `lsst.afw.math.StatisticsControl` 529 Statistics control object for coadd 530 convergenceMetric : `float` 531 Quality of fit metric for the matched templates of the input images. 532 baseMask : `lsst.afw.image.Mask` 533 Mask of the initial template coadd. 534 subfilterVariance : `list` of `numpy.ndarray` 535 The variance of each coadded subfilter image. 536 gain : `float`, optional 537 Relative weight to give the new solution when updating the model. 538 modelWeights : `numpy.ndarray` or `float` 539 A 2D array of weight values that tapers smoothly to zero away from detected sources. 540 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 543 residualGeneratorList = []
545 for tempExpRef, imageScaler, altMaskSpans
in zip(tempExpRefList, imageScalerList, spanSetMaskList):
546 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=dcrBBox)
547 visitInfo = exposure.getInfo().getVisitInfo()
548 wcs = exposure.getInfo().getWcs()
549 maskedImage = exposure.maskedImage
550 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl, visitInfo=visitInfo,
551 bbox=dcrBBox, wcs=wcs, mask=baseMask,
552 splitSubfilters=self.config.splitSubfilters)
553 imageScaler.scaleMaskedImage(maskedImage)
554 if altMaskSpans
is not None:
557 if self.config.removeMaskPlanes:
559 maskedImage -= templateImage
560 maskedImage.image.array *= modelWeights
561 residualGeneratorList.append(self.
dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
565 statsFlags, statsCtrl, weightList,
566 mask=baseMask, gain=gain)
567 dcrModels.assign(dcrSubModelOut, bbox)
570 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 574 residual : `lsst.afw.image.MaskedImageF` 575 The residual masked image for one exposure, 576 after subtracting the matched template 577 visitInfo : `lsst.afw.image.VisitInfo` 578 Metadata for the exposure. 579 bbox : `lsst.afw.geom.box.Box2I` 580 Sub-region of the coadd 581 wcs : `lsst.afw.geom.SkyWcs` 582 Coordinate system definition (wcs) for the exposure. 583 filterInfo : `lsst.afw.image.Filter` 584 The filter definition, set in the current instruments' obs package. 585 Required for any calculation of DCR, including making matched templates. 589 residualImage : `lsst.afw.image.maskedImageF` 590 The residual image for the next subfilter, shifted for DCR. 592 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
594 yield applyDcr(residual, dcr, self.
warpCtrl, bbox=bbox, useInverse=
True)
597 statsFlags, statsCtrl, weightList,
599 """Calculate a new DcrModel from a set of image residuals. 603 dcrModels : `lsst.pipe.tasks.DcrModel` 604 Current model of the true sky after correcting chromatic effects. 605 residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF` 606 The residual image for the next subfilter, shifted for DCR. 607 bbox : `lsst.afw.geom.box.Box2I` 608 Sub-region of the coadd 609 statsFlags : `lsst.afw.math.Property` 610 Statistics settings for coaddition. 611 statsCtrl : `lsst.afw.math.StatisticsControl` 612 Statistics control object for coadd 613 weightList : `list` of `float` 614 The weight to give each input exposure in the coadd 615 mask : `lsst.afw.image.Mask` 616 Mask to use for each new model image. 618 Relative weight to give the new solution when updating the model. 622 dcrModel : `lsst.pipe.tasks.DcrModel` 623 New model of the true sky after correcting chromatic effects. 626 clipped = dcrModels.mask.getPlaneBitMask(
"CLIPPED")
628 for subfilter, model
in enumerate(dcrModels):
629 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
630 residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
632 residual.setXY0(bbox.getBegin())
634 residual += model[bbox]
637 badPixels = ~np.isfinite(newModel.image.array)
640 newModel.setMask(mask[bbox])
641 newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
642 if self.config.regularizeModelIterations > 0:
643 dcrModels.regularizeModelIter(subfilter, newModel, bbox,
644 self.config.regularizeModelIterations,
645 self.config.regularizationWidth)
646 newModelImages.append(newModel)
647 if self.config.regularizeModelFrequency > 0:
648 dcrModels.regularizeModelFreq(newModelImages, bbox,
649 self.config.regularizeModelFrequency,
650 self.config.regularizationWidth)
651 dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
652 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
655 weightList, spanSetMaskList, statsCtrl):
656 """Calculate a quality of fit metric for the matched templates. 660 dcrModels : `lsst.pipe.tasks.DcrModel` 661 Best fit model of the true sky after correcting chromatic effects. 662 bbox : `lsst.afw.geom.box.Box2I` 664 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 665 The data references to the input warped exposures. 666 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 667 The image scalars correct for the zero point of the exposures. 668 weightList : `list` of `float` 669 The weight to give each input exposure in the coadd 670 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 671 Each element is dict with keys = mask plane name to add the spans to 672 statsCtrl : `lsst.afw.math.StatisticsControl` 673 Statistics control object for coadd 677 convergenceMetric : `float` 678 Quality of fit metric for all input exposures, within the sub-region 680 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
682 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
688 zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
689 for tempExpRef, expWeight, imageScaler, altMaskSpans
in zipIterables:
690 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
691 imageScaler.scaleMaskedImage(exposure.maskedImage)
693 altMaskSpans=altMaskSpans)
694 metric += singleMetric*expWeight
695 metricList[tempExpRef.dataId[
"visit"]] = singleMetric
697 self.log.info(
"Individual metrics:\n%s", metricList)
698 return 1.0
if weight == 0.0
else metric/weight
701 statsCtrl, altMaskSpans=None):
702 """Calculate a quality of fit metric for a single matched template. 706 dcrModels : `lsst.pipe.tasks.DcrModel` 707 Best fit model of the true sky after correcting chromatic effects. 708 exposure : `lsst.afw.image.ExposureF` 709 The input warped exposure to evaluate. 710 significanceImage : `numpy.ndarray` 711 Array of weights for each pixel corresponding to its significance 712 for the convergence calculation. 713 statsCtrl : `lsst.afw.math.StatisticsControl` 714 Statistics control object for coadd 715 altMaskSpans : `dict` containing spanSet lists, or None 716 The keys of the `dict` equal the mask plane name to add the spans to 720 convergenceMetric : `float` 721 Quality of fit metric for one exposure, within the sub-region. 723 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
724 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl,
725 visitInfo=exposure.getInfo().getVisitInfo(),
726 bbox=exposure.getBBox(),
727 wcs=exposure.getInfo().getWcs())
728 diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
729 refVals = np.abs(templateImage.image.array)*significanceImage
731 finitePixels = np.isfinite(diffVals)
732 if altMaskSpans
is not None:
734 goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
735 convergeMaskPixels = exposure.mask.array & convergeMask > 0
736 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
737 if np.sum(usePixels) == 0:
740 diffUse = diffVals[usePixels]
741 refUse = refVals[usePixels]
742 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
746 """Add a list of sub-band coadds together. 750 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 751 A list of coadd exposures, each exposure containing 752 the model for one subfilter. 756 coaddExposure : `lsst.afw.image.ExposureF` 757 A single coadd exposure that is the sum of the sub-bands. 759 coaddExposure = dcrCoadds[0].clone()
760 for coadd
in dcrCoadds[1:]:
761 coaddExposure.maskedImage += coadd.maskedImage
764 def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
766 """Create a list of coadd exposures from a list of masked images. 770 dcrModels : `lsst.pipe.tasks.DcrModel` 771 Best fit model of the true sky after correcting chromatic effects. 772 skyInfo : `lsst.pipe.base.Struct` 773 Patch geometry information, from getSkyInfo 774 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 775 The data references to the input warped exposures. 776 weightList : `list` of `float` 777 The weight to give each input exposure in the coadd 778 calibration : `lsst.afw.Image.Calib`, optional 779 Scale factor to set the photometric zero point of an exposure. 780 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 781 A record of the observations that are included in the coadd. 782 mask : `lsst.afw.image.Mask`, optional 783 Optional mask to override the values in the final coadd. 787 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 788 A list of coadd exposures, each exposure containing 789 the model for one subfilter. 792 for model
in dcrModels:
793 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
794 if calibration
is not None:
795 coaddExposure.setCalib(calibration)
796 if coaddInputs
is not None:
797 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
800 coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
801 coaddExposure.setMaskedImage(model[skyInfo.bbox])
803 coaddExposure.setMask(mask)
804 dcrCoadds.append(coaddExposure)
808 """Calculate the gain to use for the current iteration. 810 After calculating a new DcrModel, each value is averaged with the 811 value in the corresponding pixel from the previous iteration. This 812 reduces oscillating solutions that iterative techniques are plagued by, 813 and speeds convergence. By far the biggest changes to the model 814 happen in the first couple iterations, so we can also use a more 815 aggressive gain later when the model is changing slowly. 819 convergenceList : `list` of `float` 820 The quality of fit metric from each previous iteration. 821 gainList : `list` of `float` 822 The gains used in each previous iteration: appended with the new 824 Gains are numbers between ``self.config.baseGain`` and 1. 829 Relative weight to give the new solution when updating the model. 830 A value of 1.0 gives equal weight to both solutions. 835 If ``len(convergenceList) != len(gainList)+1``. 837 nIter = len(convergenceList)
838 if nIter != len(gainList) + 1:
839 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 840 % (len(convergenceList), len(gainList)))
842 if self.config.baseGain
is None:
845 baseGain = 1./(self.config.dcrNumSubfilters - 1)
847 baseGain = self.config.baseGain
849 if self.config.useProgressiveGain
and nIter > 2:
857 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
858 for i
in range(nIter - 1)]
861 estFinalConv = np.array(estFinalConv)
862 estFinalConv[estFinalConv < 0] = 0
864 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
865 lastGain = gainList[-1]
866 lastConv = convergenceList[-2]
867 newConv = convergenceList[-1]
872 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
878 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
879 newGain = 1 - abs(delta)
881 newGain = (newGain + lastGain)/2.
882 gain = max(baseGain, newGain)
885 gainList.append(gain)
889 """Build an array that smoothly tapers to 0 away from detected sources. 893 dcrModels : `lsst.pipe.tasks.DcrModel` 894 Best fit model of the true sky after correcting chromatic effects. 895 dcrBBox : `lsst.afw.geom.box.Box2I` 896 Sub-region of the coadd which includes a buffer to allow for DCR. 900 weights : `numpy.ndarray` or `float` 901 A 2D array of weight values that tapers smoothly to zero away from detected sources. 902 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 907 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 909 if self.config.modelWeightsWidth < 0:
910 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
911 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
912 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
913 weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
914 weights[convergeMaskPixels] = 1.
915 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
916 weights /= np.max(weights)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def removeMaskPlanes(self, maskedImage)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def setRejectedMaskMapping(statsCtrl)
def applyAltEdgeMask(self, mask, altMaskList)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)
def _subBBoxIter(bbox, subregionSize)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)