25 from scipy
import ndimage
33 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
35 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
39 dcrNumSubfilters = pexConfig.Field(
41 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
44 maxNumIter = pexConfig.Field(
46 doc=
"Maximum number of iterations of forward modeling.",
49 minNumIter = pexConfig.Field(
51 doc=
"Minimum number of iterations of forward modeling.",
54 convergenceThreshold = pexConfig.Field(
56 doc=
"Target relative change in convergence between iterations of forward modeling.",
59 useConvergence = pexConfig.Field(
61 doc=
"Use convergence test as a forward modeling end condition?" 62 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
65 baseGain = pexConfig.Field(
68 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 69 "A value of 1.0 gives equal weight to both solutions." 70 "Small values imply slower convergence of the solution, but can " 71 "help prevent overshooting and failures in the fit." 72 "If ``baseGain`` is None, a conservative gain " 73 "will be calculated from the number of subfilters. ",
76 useProgressiveGain = pexConfig.Field(
78 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 79 "When calculating the next gain, we use up to 5 previous gains and convergence values." 80 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
83 doAirmassWeight = pexConfig.Field(
85 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
88 modelWeightsWidth = pexConfig.Field(
90 doc=
"Width of the region around detected sources to include in the DcrModel.",
93 useModelWeights = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 splitSubfilters = pexConfig.Field(
100 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 101 "Instead of at the midpoint",
104 regularizeModelIterations = pexConfig.Field(
106 doc=
"Maximum relative change of the model allowed between iterations." 107 "Set to zero to disable.",
110 regularizeModelFrequency = pexConfig.Field(
112 doc=
"Maximum relative change of the model allowed between subfilters." 113 "Set to zero to disable.",
116 convergenceMaskPlanes = pexConfig.ListField(
118 default=[
"DETECTED"],
119 doc=
"Mask planes to use to calculate convergence." 121 regularizationWidth = pexConfig.Field(
124 doc=
"Minimum radius of a region to include in regularization, in pixels." 126 imageWarpMethod = pexConfig.Field(
128 doc=
"Name of the warping kernel to use for shifting the image and variance planes.",
131 maskWarpMethod = pexConfig.Field(
133 doc=
"Name of the warping kernel to use for shifting the mask plane.",
138 CompareWarpAssembleCoaddConfig.setDefaults(self)
150 """Assemble DCR coadded images from a set of warps. 155 The number of pixels to grow each subregion by to allow for DCR. 156 warpCtrl : `lsst.afw.math.WarpingControl` 157 Configuration settings for warping an image 161 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 162 Warps (also called coadded temporary exposures), including the effects of 163 Differential Chromatic Refraction (DCR). 164 For full details of the mathematics and algorithm, please see 165 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 167 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 168 each subfilter used in the iterative calculation. 169 It begins by dividing the bandpass-defining filter into N equal bandwidth 170 "subfilters", and divides the flux in each pixel from an initial coadd 171 equally into each as a "dcrModel". Because the airmass and parallactic 172 angle of each individual exposure is known, we can calculate the shift 173 relative to the center of the band in each subfilter due to DCR. For each 174 exposure we apply this shift as a linear transformation to the dcrModels 175 and stack the results to produce a DCR-matched exposure. The matched 176 exposures are subtracted from the input exposures to produce a set of 177 residual images, and these residuals are reverse shifted for each 178 exposures' subfilters and stacked. The shifted and stacked residuals are 179 added to the dcrModels to produce a new estimate of the flux in each pixel 180 within each subfilter. The dcrModels are solved for iteratively, which 181 continues until the solution from a new iteration improves by less than 182 a set percentage, or a maximum number of iterations is reached. 183 Two forms of regularization are employed to reduce unphysical results. 184 First, the new solution is averaged with the solution from the previous 185 iteration, which mitigates oscillating solutions where the model 186 overshoots with alternating very high and low values. 187 Second, a common degeneracy when the data have a limited range of airmass or 188 parallactic angle values is for one subfilter to be fit with very low or 189 negative values, while another subfilter is fit with very high values. This 190 typically appears in the form of holes next to sources in one subfilter, 191 and corresponding extended wings in another. Because each subfilter has 192 a narrow bandwidth we assume that physical sources that are above the noise 193 level will not vary in flux by more than a factor of `frequencyClampFactor` 194 between subfilters, and pixels that have flux deviations larger than that 195 factor will have the excess flux distributed evenly among all subfilters. 198 ConfigClass = DcrAssembleCoaddConfig
199 _DefaultName =
"dcrAssembleCoadd" 203 """Assemble a coadd from a set of warps. 205 Coadd a set of Warps. Compute weights to be applied to each Warp and 206 find scalings to match the photometric zeropoint to a reference Warp. 207 Assemble the Warps using run method. 208 Forward model chromatic effects across multiple subfilters, 209 and subtract from the input Warps to build sets of residuals. 210 Use the residuals to construct a new ``DcrModel`` for each subfilter, 211 and iterate until the model converges. 212 Interpolate over NaNs and optionally write the coadd to disk. 213 Return the coadded exposure. 217 dataRef : `lsst.daf.persistence.ButlerDataRef` 218 Data reference defining the patch for coaddition and the 220 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 221 List of data references to warps. Data to be coadded will be 222 selected from this list based on overlap with the patch defined by 227 results : `lsst.pipe.base.Struct` 228 The Struct contains the following fields: 230 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 231 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 232 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 233 - ``dcrNImages``: `list` of exposure count images for each subfilter 235 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList)
236 for subfilter
in range(self.config.dcrNumSubfilters):
238 if self.config.doWrite:
239 self.log.info(
"Persisting dcrCoadd")
240 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
241 numSubfilters=self.config.dcrNumSubfilters)
242 if self.config.doNImage
and results.dcrNImages
is not None:
243 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
244 numSubfilters=self.config.dcrNumSubfilters)
249 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 251 Sets the properties ``warpCtrl`` and ``bufferSize``. 255 templateCoadd : `lsst.afw.image.ExposureF` 256 The initial coadd exposure before accounting for DCR. 257 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 258 The data references to the input warped exposures. 259 weightList : `list` of `float` 260 The weight to give each input exposure in the coadd 261 Will be modified in place if ``doAirmassWeight`` is set. 265 dcrModels : `lsst.pipe.tasks.DcrModel` 266 Best fit model of the true sky after correcting chromatic effects. 271 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 273 filterInfo = templateCoadd.getFilter()
274 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
275 raise NotImplementedError(
"No minimum/maximum wavelength information found" 276 " in the filter definition! Please add lambdaMin and lambdaMax" 277 " to the Mapper class in your obs package.")
280 for visitNum, tempExpRef
in enumerate(tempExpRefList):
281 visitInfo = tempExpRef.get(tempExpName +
"_visitInfo")
282 airmass = visitInfo.getBoresightAirmass()
283 if self.config.doAirmassWeight:
284 weightList[visitNum] *= airmass
285 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
286 filterInfo, self.config.dcrNumSubfilters))))
291 warpInterpLength = max(self.config.subregionSize)
292 self.
warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
293 self.config.maskWarpMethod,
294 cacheSize=warpCache, interpLength=warpInterpLength)
295 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
296 self.config.dcrNumSubfilters,
297 filterInfo=filterInfo,
298 psf=templateCoadd.getPsf())
301 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
302 supplementaryData=None):
303 """Assemble the coadd. 305 Requires additional inputs Struct ``supplementaryData`` to contain a 306 ``templateCoadd`` that serves as the model of the static sky. 308 Find artifacts and apply them to the warps' masks creating a list of 309 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 310 Then pass these alternative masks to the base class's assemble method. 312 Divide the ``templateCoadd`` evenly between each subfilter of a 313 ``DcrModel`` as the starting best estimate of the true wavelength- 314 dependent sky. Forward model the ``DcrModel`` using the known 315 chromatic effects in each subfilter and calculate a convergence metric 316 based on how well the modeled template matches the input warps. If 317 the convergence has not yet reached the desired threshold, then shift 318 and stack the residual images to build a new ``DcrModel``. Apply 319 conditioning to prevent oscillating solutions between iterations or 322 Once the ``DcrModel`` reaches convergence or the maximum number of 323 iterations has been reached, fill the metadata for each subfilter 324 image and make them proper ``coaddExposure``s. 328 skyInfo : `lsst.pipe.base.Struct` 329 Patch geometry information, from getSkyInfo 330 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 331 The data references to the input warped exposures. 332 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 333 The image scalars correct for the zero point of the exposures. 334 weightList : `list` of `float` 335 The weight to give each input exposure in the coadd 336 supplementaryData : `lsst.pipe.base.Struct` 337 Result struct returned by ``makeSupplementaryData`` with components: 339 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 343 result : `lsst.pipe.base.Struct` 344 Result struct with components: 346 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 347 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 348 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 349 - ``dcrNImages``: `list` of exposure count images for each subfilter 351 templateCoadd = supplementaryData.templateCoadd
352 baseMask = templateCoadd.mask.clone()
355 baseVariance = templateCoadd.variance.clone()
356 baseVariance /= self.config.dcrNumSubfilters
357 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
359 templateCoadd.setMask(baseMask)
360 badMaskPlanes = self.config.badMaskPlanes[:]
361 badMaskPlanes.append(
"CLIPPED")
362 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
365 dcrModels = self.
prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
366 if self.config.doNImage:
368 tempExpRefList, spanSetMaskList, stats.ctrl)
369 nImage = afwImage.ImageU(skyInfo.bbox)
373 for dcrNImage
in dcrNImages:
378 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
379 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
380 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
382 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
385 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
386 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
387 dcrBBox = afwGeom.Box2I(subBBox)
389 dcrBBox.clip(dcrModels.bbox)
390 if self.config.useModelWeights:
395 imageScalerList, weightList, spanSetMaskList,
397 self.log.info(
"Initial convergence : %s", convergenceMetric)
398 convergenceList = [convergenceMetric]
400 convergenceCheck = 1.
401 subfilterVariance =
None 402 while (convergenceCheck > self.config.convergenceThreshold
or 403 modelIter < self.config.minNumIter):
406 weightList, spanSetMaskList, stats.flags, stats.ctrl,
407 convergenceMetric, baseMask, subfilterVariance, gain,
409 if self.config.useConvergence:
411 imageScalerList, weightList,
414 if convergenceMetric == 0:
415 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 416 "most likely due to there being no valid data in the region.",
417 skyInfo.patchInfo.getIndex(), subIter)
419 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
420 if convergenceCheck < 0:
421 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 422 "iterations or desired convergence improvement of %s." 424 skyInfo.patchInfo.getIndex(), subIter,
425 self.config.convergenceThreshold, convergenceCheck)
427 convergenceList.append(convergenceMetric)
428 if modelIter > self.config.maxNumIter:
429 if self.config.useConvergence:
430 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 431 "before reaching desired convergence improvement of %s." 432 " Final convergence improvement: %s",
433 skyInfo.patchInfo.getIndex(), subIter,
434 self.config.convergenceThreshold, convergenceCheck)
437 if self.config.useConvergence:
438 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
439 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
442 if self.config.useConvergence:
443 self.log.info(
"Coadd patch %s subregion %s finished with " 444 "convergence metric %s after %s iterations",
445 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
447 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
448 skyInfo.patchInfo.getIndex(), subIter, modelIter)
449 if self.config.useConvergence
and convergenceMetric > 0:
450 self.log.info(
"Final convergence improvement was %.4f%% overall",
451 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
453 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
454 calibration=self.scaleZeroPoint.getCalib(),
455 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
457 variance=baseVariance)
459 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
460 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
462 def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
463 """Calculate the number of exposures contributing to each subfilter. 467 dcrModels : `lsst.pipe.tasks.DcrModel` 468 Best fit model of the true sky after correcting chromatic effects. 469 bbox : `lsst.afw.geom.box.Box2I` 470 Bounding box of the patch to coadd. 471 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 472 The data references to the input warped exposures. 473 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 474 Each element is dict with keys = mask plane name to add the spans to 475 statsCtrl : `lsst.afw.math.StatisticsControl` 476 Statistics control object for coadd 480 dcrNImages : `list` of `lsst.afw.image.ImageU` 481 List of exposure count images for each subfilter 483 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
485 for tempExpRef, altMaskSpans
in zip(tempExpRefList, spanSetMaskList):
486 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
487 visitInfo = exposure.getInfo().getVisitInfo()
488 wcs = exposure.getInfo().getWcs()
490 if altMaskSpans
is not None:
492 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
493 for dcr, dcrNImage
in zip(dcrShift, dcrNImages):
494 shiftedImage = applyDcr(exposure.maskedImage, dcr, self.
warpCtrl, useInverse=
True)
495 dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
498 def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
499 spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
500 baseMask, subfilterVariance, gain, modelWeights):
501 """Assemble the DCR coadd for a sub-region. 503 Build a DCR-matched template for each input exposure, then shift the 504 residuals according to the DCR in each subfilter. 505 Stack the shifted residuals and apply them as a correction to the 506 solution from the previous iteration. 507 Restrict the new model solutions from varying by more than a factor of 508 `modelClampFactor` from the last solution, and additionally restrict the 509 individual subfilter models from varying by more than a factor of 510 `frequencyClampFactor` from their average. 511 Finally, mitigate potentially oscillating solutions by averaging the new 512 solution with the solution from the previous iteration, weighted by 513 their convergence metric. 517 dcrModels : `lsst.pipe.tasks.DcrModel` 518 Best fit model of the true sky after correcting chromatic effects. 519 bbox : `lsst.afw.geom.box.Box2I` 520 Bounding box of the subregion to coadd. 521 dcrBBox :`lsst.afw.geom.box.Box2I` 522 Sub-region of the coadd which includes a buffer to allow for DCR. 523 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 524 The data references to the input warped exposures. 525 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 526 The image scalars correct for the zero point of the exposures. 527 weightList : `list` of `float` 528 The weight to give each input exposure in the coadd 529 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 530 Each element is dict with keys = mask plane name to add the spans to 531 statsFlags : `lsst.afw.math.Property` 532 Statistics settings for coaddition. 533 statsCtrl : `lsst.afw.math.StatisticsControl` 534 Statistics control object for coadd 535 convergenceMetric : `float` 536 Quality of fit metric for the matched templates of the input images. 537 baseMask : `lsst.afw.image.Mask` 538 Mask of the initial template coadd. 539 subfilterVariance : `list` of `numpy.ndarray` 540 The variance of each coadded subfilter image. 541 gain : `float`, optional 542 Relative weight to give the new solution when updating the model. 543 modelWeights : `numpy.ndarray` or `float` 544 A 2D array of weight values that tapers smoothly to zero away from detected sources. 545 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 548 residualGeneratorList = []
550 for tempExpRef, imageScaler, altMaskSpans
in zip(tempExpRefList, imageScalerList, spanSetMaskList):
551 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=dcrBBox)
552 visitInfo = exposure.getInfo().getVisitInfo()
553 wcs = exposure.getInfo().getWcs()
554 maskedImage = exposure.maskedImage
555 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl, visitInfo=visitInfo,
556 bbox=dcrBBox, wcs=wcs, mask=baseMask,
557 splitSubfilters=self.config.splitSubfilters)
558 imageScaler.scaleMaskedImage(maskedImage)
559 if altMaskSpans
is not None:
562 if self.config.removeMaskPlanes:
564 maskedImage -= templateImage
565 maskedImage.image.array *= modelWeights
566 residualGeneratorList.append(self.
dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
570 statsFlags, statsCtrl, weightList,
571 mask=baseMask, gain=gain)
572 dcrModels.assign(dcrSubModelOut, bbox)
575 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 579 residual : `lsst.afw.image.MaskedImageF` 580 The residual masked image for one exposure, 581 after subtracting the matched template 582 visitInfo : `lsst.afw.image.VisitInfo` 583 Metadata for the exposure. 584 bbox : `lsst.afw.geom.box.Box2I` 585 Sub-region of the coadd 586 wcs : `lsst.afw.geom.SkyWcs` 587 Coordinate system definition (wcs) for the exposure. 588 filterInfo : `lsst.afw.image.Filter` 589 The filter definition, set in the current instruments' obs package. 590 Required for any calculation of DCR, including making matched templates. 594 residualImage : `lsst.afw.image.maskedImageF` 595 The residual image for the next subfilter, shifted for DCR. 597 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
599 yield applyDcr(residual, dcr, self.
warpCtrl, bbox=bbox, useInverse=
True)
602 statsFlags, statsCtrl, weightList,
604 """Calculate a new DcrModel from a set of image residuals. 608 dcrModels : `lsst.pipe.tasks.DcrModel` 609 Current model of the true sky after correcting chromatic effects. 610 residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF` 611 The residual image for the next subfilter, shifted for DCR. 612 bbox : `lsst.afw.geom.box.Box2I` 613 Sub-region of the coadd 614 statsFlags : `lsst.afw.math.Property` 615 Statistics settings for coaddition. 616 statsCtrl : `lsst.afw.math.StatisticsControl` 617 Statistics control object for coadd 618 weightList : `list` of `float` 619 The weight to give each input exposure in the coadd 620 mask : `lsst.afw.image.Mask` 621 Mask to use for each new model image. 623 Relative weight to give the new solution when updating the model. 627 dcrModel : `lsst.pipe.tasks.DcrModel` 628 New model of the true sky after correcting chromatic effects. 631 clipped = dcrModels.mask.getPlaneBitMask(
"CLIPPED")
633 for subfilter, model
in enumerate(dcrModels):
634 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
635 residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
637 residual.setXY0(bbox.getBegin())
639 residual += model[bbox]
642 badPixels = ~np.isfinite(newModel.image.array)
645 newModel.setMask(mask[bbox])
646 newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
647 if self.config.regularizeModelIterations > 0:
648 dcrModels.regularizeModelIter(subfilter, newModel, bbox,
649 self.config.regularizeModelIterations,
650 self.config.regularizationWidth)
651 newModelImages.append(newModel)
652 if self.config.regularizeModelFrequency > 0:
653 dcrModels.regularizeModelFreq(newModelImages, bbox,
654 self.config.regularizeModelFrequency,
655 self.config.regularizationWidth)
656 dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
657 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
660 weightList, spanSetMaskList, statsCtrl):
661 """Calculate a quality of fit metric for the matched templates. 665 dcrModels : `lsst.pipe.tasks.DcrModel` 666 Best fit model of the true sky after correcting chromatic effects. 667 bbox : `lsst.afw.geom.box.Box2I` 669 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 670 The data references to the input warped exposures. 671 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 672 The image scalars correct for the zero point of the exposures. 673 weightList : `list` of `float` 674 The weight to give each input exposure in the coadd 675 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 676 Each element is dict with keys = mask plane name to add the spans to 677 statsCtrl : `lsst.afw.math.StatisticsControl` 678 Statistics control object for coadd 682 convergenceMetric : `float` 683 Quality of fit metric for all input exposures, within the sub-region 685 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
687 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
693 zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
694 for tempExpRef, expWeight, imageScaler, altMaskSpans
in zipIterables:
695 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
696 imageScaler.scaleMaskedImage(exposure.maskedImage)
698 altMaskSpans=altMaskSpans)
699 metric += singleMetric*expWeight
700 metricList[tempExpRef.dataId[
"visit"]] = singleMetric
702 self.log.info(
"Individual metrics:\n%s", metricList)
703 return 1.0
if weight == 0.0
else metric/weight
706 statsCtrl, altMaskSpans=None):
707 """Calculate a quality of fit metric for a single matched template. 711 dcrModels : `lsst.pipe.tasks.DcrModel` 712 Best fit model of the true sky after correcting chromatic effects. 713 exposure : `lsst.afw.image.ExposureF` 714 The input warped exposure to evaluate. 715 significanceImage : `numpy.ndarray` 716 Array of weights for each pixel corresponding to its significance 717 for the convergence calculation. 718 statsCtrl : `lsst.afw.math.StatisticsControl` 719 Statistics control object for coadd 720 altMaskSpans : `dict` containing spanSet lists, or None 721 The keys of the `dict` equal the mask plane name to add the spans to 725 convergenceMetric : `float` 726 Quality of fit metric for one exposure, within the sub-region. 728 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
729 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl,
730 visitInfo=exposure.getInfo().getVisitInfo(),
731 bbox=exposure.getBBox(),
732 wcs=exposure.getInfo().getWcs())
733 diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
734 refVals = np.abs(templateImage.image.array)*significanceImage
736 finitePixels = np.isfinite(diffVals)
737 if altMaskSpans
is not None:
739 goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
740 convergeMaskPixels = exposure.mask.array & convergeMask > 0
741 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
742 if np.sum(usePixels) == 0:
745 diffUse = diffVals[usePixels]
746 refUse = refVals[usePixels]
747 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
751 """Add a list of sub-band coadds together. 755 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 756 A list of coadd exposures, each exposure containing 757 the model for one subfilter. 761 coaddExposure : `lsst.afw.image.ExposureF` 762 A single coadd exposure that is the sum of the sub-bands. 764 coaddExposure = dcrCoadds[0].clone()
765 for coadd
in dcrCoadds[1:]:
766 coaddExposure.maskedImage += coadd.maskedImage
769 def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
770 mask=None, variance=None):
771 """Create a list of coadd exposures from a list of masked images. 775 dcrModels : `lsst.pipe.tasks.DcrModel` 776 Best fit model of the true sky after correcting chromatic effects. 777 skyInfo : `lsst.pipe.base.Struct` 778 Patch geometry information, from getSkyInfo 779 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 780 The data references to the input warped exposures. 781 weightList : `list` of `float` 782 The weight to give each input exposure in the coadd 783 calibration : `lsst.afw.Image.Calib`, optional 784 Scale factor to set the photometric zero point of an exposure. 785 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 786 A record of the observations that are included in the coadd. 787 mask : `lsst.afw.image.Mask`, optional 788 Optional mask to override the values in the final coadd. 789 variance : `lsst.afw.image.Image`, optional 790 Optional variance plane to override the values in the final coadd. 794 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 795 A list of coadd exposures, each exposure containing 796 the model for one subfilter. 799 for model
in dcrModels:
800 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
801 if calibration
is not None:
802 coaddExposure.setCalib(calibration)
803 if coaddInputs
is not None:
804 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
807 coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
808 coaddExposure.setMaskedImage(model[skyInfo.bbox])
810 coaddExposure.setMask(mask)
811 if variance
is not None:
812 coaddExposure.setVariance(variance)
813 dcrCoadds.append(coaddExposure)
817 """Calculate the gain to use for the current iteration. 819 After calculating a new DcrModel, each value is averaged with the 820 value in the corresponding pixel from the previous iteration. This 821 reduces oscillating solutions that iterative techniques are plagued by, 822 and speeds convergence. By far the biggest changes to the model 823 happen in the first couple iterations, so we can also use a more 824 aggressive gain later when the model is changing slowly. 828 convergenceList : `list` of `float` 829 The quality of fit metric from each previous iteration. 830 gainList : `list` of `float` 831 The gains used in each previous iteration: appended with the new 833 Gains are numbers between ``self.config.baseGain`` and 1. 838 Relative weight to give the new solution when updating the model. 839 A value of 1.0 gives equal weight to both solutions. 844 If ``len(convergenceList) != len(gainList)+1``. 846 nIter = len(convergenceList)
847 if nIter != len(gainList) + 1:
848 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 849 % (len(convergenceList), len(gainList)))
851 if self.config.baseGain
is None:
854 baseGain = 1./(self.config.dcrNumSubfilters - 1)
856 baseGain = self.config.baseGain
858 if self.config.useProgressiveGain
and nIter > 2:
866 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
867 for i
in range(nIter - 1)]
870 estFinalConv = np.array(estFinalConv)
871 estFinalConv[estFinalConv < 0] = 0
873 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
874 lastGain = gainList[-1]
875 lastConv = convergenceList[-2]
876 newConv = convergenceList[-1]
881 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
887 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
888 newGain = 1 - abs(delta)
890 newGain = (newGain + lastGain)/2.
891 gain = max(baseGain, newGain)
894 gainList.append(gain)
898 """Build an array that smoothly tapers to 0 away from detected sources. 902 dcrModels : `lsst.pipe.tasks.DcrModel` 903 Best fit model of the true sky after correcting chromatic effects. 904 dcrBBox : `lsst.afw.geom.box.Box2I` 905 Sub-region of the coadd which includes a buffer to allow for DCR. 909 weights : `numpy.ndarray` or `float` 910 A 2D array of weight values that tapers smoothly to zero away from detected sources. 911 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 916 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 918 if self.config.modelWeightsWidth < 0:
919 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
920 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
921 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
922 weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
923 weights[convergeMaskPixels] = 1.
924 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
925 weights /= np.max(weights)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def runDataRef(self, dataRef, selectDataList=[])
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def removeMaskPlanes(self, maskedImage)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def setRejectedMaskMapping(statsCtrl)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)
def _subBBoxIter(bbox, subregionSize)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)