25 from scipy
import ndimage
33 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
35 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
39 dcrNumSubfilters = pexConfig.Field(
41 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
44 maxNumIter = pexConfig.Field(
46 doc=
"Maximum number of iterations of forward modeling.",
49 minNumIter = pexConfig.Field(
51 doc=
"Minimum number of iterations of forward modeling.",
54 convergenceThreshold = pexConfig.Field(
56 doc=
"Target relative change in convergence between iterations of forward modeling.",
59 useConvergence = pexConfig.Field(
61 doc=
"Use convergence test as a forward modeling end condition?" 62 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
65 baseGain = pexConfig.Field(
68 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 69 "A value of 1.0 gives equal weight to both solutions." 70 "Small values imply slower convergence of the solution, but can " 71 "help prevent overshooting and failures in the fit." 72 "If ``baseGain`` is None, a conservative gain " 73 "will be calculated from the number of subfilters. ",
76 useProgressiveGain = pexConfig.Field(
78 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 79 "When calculating the next gain, we use up to 5 previous gains and convergence values." 80 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
83 doAirmassWeight = pexConfig.Field(
85 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
88 modelWeightsWidth = pexConfig.Field(
90 doc=
"Width of the region around detected sources to include in the DcrModel.",
93 useModelWeights = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 splitSubfilters = pexConfig.Field(
100 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 101 "Instead of at the midpoint",
104 regularizeModelIterations = pexConfig.Field(
106 doc=
"Maximum relative change of the model allowed between iterations." 107 "Set to zero to disable.",
110 regularizeModelFrequency = pexConfig.Field(
112 doc=
"Maximum relative change of the model allowed between subfilters." 113 "Set to zero to disable.",
116 convergenceMaskPlanes = pexConfig.ListField(
118 default=[
"DETECTED"],
119 doc=
"Mask planes to use to calculate convergence." 121 regularizationWidth = pexConfig.Field(
124 doc=
"Minimum radius of a region to include in regularization, in pixels." 126 imageWarpMethod = pexConfig.Field(
128 doc=
"Name of the warping kernel to use for shifting the image and variance planes.",
131 maskWarpMethod = pexConfig.Field(
133 doc=
"Name of the warping kernel to use for shifting the mask plane.",
138 CompareWarpAssembleCoaddConfig.setDefaults(self)
150 """Assemble DCR coadded images from a set of warps. 155 The number of pixels to grow each subregion by to allow for DCR. 156 warpCtrl : `lsst.afw.math.WarpingControl` 157 Configuration settings for warping an image 161 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 162 Warps (also called coadded temporary exposures), including the effects of 163 Differential Chromatic Refraction (DCR). 164 For full details of the mathematics and algorithm, please see 165 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 167 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 168 each subfilter used in the iterative calculation. 169 It begins by dividing the bandpass-defining filter into N equal bandwidth 170 "subfilters", and divides the flux in each pixel from an initial coadd 171 equally into each as a "dcrModel". Because the airmass and parallactic 172 angle of each individual exposure is known, we can calculate the shift 173 relative to the center of the band in each subfilter due to DCR. For each 174 exposure we apply this shift as a linear transformation to the dcrModels 175 and stack the results to produce a DCR-matched exposure. The matched 176 exposures are subtracted from the input exposures to produce a set of 177 residual images, and these residuals are reverse shifted for each 178 exposures' subfilters and stacked. The shifted and stacked residuals are 179 added to the dcrModels to produce a new estimate of the flux in each pixel 180 within each subfilter. The dcrModels are solved for iteratively, which 181 continues until the solution from a new iteration improves by less than 182 a set percentage, or a maximum number of iterations is reached. 183 Two forms of regularization are employed to reduce unphysical results. 184 First, the new solution is averaged with the solution from the previous 185 iteration, which mitigates oscillating solutions where the model 186 overshoots with alternating very high and low values. 187 Second, a common degeneracy when the data have a limited range of airmass or 188 parallactic angle values is for one subfilter to be fit with very low or 189 negative values, while another subfilter is fit with very high values. This 190 typically appears in the form of holes next to sources in one subfilter, 191 and corresponding extended wings in another. Because each subfilter has 192 a narrow bandwidth we assume that physical sources that are above the noise 193 level will not vary in flux by more than a factor of `frequencyClampFactor` 194 between subfilters, and pixels that have flux deviations larger than that 195 factor will have the excess flux distributed evenly among all subfilters. 198 ConfigClass = DcrAssembleCoaddConfig
199 _DefaultName =
"dcrAssembleCoadd" 202 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
203 """Assemble a coadd from a set of warps. 205 Coadd a set of Warps. Compute weights to be applied to each Warp and 206 find scalings to match the photometric zeropoint to a reference Warp. 207 Assemble the Warps using run method. 208 Forward model chromatic effects across multiple subfilters, 209 and subtract from the input Warps to build sets of residuals. 210 Use the residuals to construct a new ``DcrModel`` for each subfilter, 211 and iterate until the model converges. 212 Interpolate over NaNs and optionally write the coadd to disk. 213 Return the coadded exposure. 217 dataRef : `lsst.daf.persistence.ButlerDataRef` 218 Data reference defining the patch for coaddition and the 220 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 221 List of data references to warps. Data to be coadded will be 222 selected from this list based on overlap with the patch defined by 227 results : `lsst.pipe.base.Struct` 228 The Struct contains the following fields: 230 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 231 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 232 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 233 - ``dcrNImages``: `list` of exposure count images for each subfilter 235 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
236 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
238 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
239 warpRefList=warpRefList)
240 for subfilter
in range(self.config.dcrNumSubfilters):
242 if self.config.doWrite:
243 self.log.info(
"Persisting dcrCoadd")
244 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
245 numSubfilters=self.config.dcrNumSubfilters)
246 if self.config.doNImage
and results.dcrNImages
is not None:
247 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
248 numSubfilters=self.config.dcrNumSubfilters)
253 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 255 Sets the properties ``warpCtrl`` and ``bufferSize``. 259 templateCoadd : `lsst.afw.image.ExposureF` 260 The initial coadd exposure before accounting for DCR. 261 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 262 The data references to the input warped exposures. 263 weightList : `list` of `float` 264 The weight to give each input exposure in the coadd 265 Will be modified in place if ``doAirmassWeight`` is set. 269 dcrModels : `lsst.pipe.tasks.DcrModel` 270 Best fit model of the true sky after correcting chromatic effects. 275 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 277 filterInfo = templateCoadd.getFilter()
278 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
279 raise NotImplementedError(
"No minimum/maximum wavelength information found" 280 " in the filter definition! Please add lambdaMin and lambdaMax" 281 " to the Mapper class in your obs package.")
284 for visitNum, tempExpRef
in enumerate(tempExpRefList):
285 visitInfo = tempExpRef.get(tempExpName +
"_visitInfo")
286 airmass = visitInfo.getBoresightAirmass()
287 if self.config.doAirmassWeight:
288 weightList[visitNum] *= airmass
289 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
290 filterInfo, self.config.dcrNumSubfilters))))
295 warpInterpLength = max(self.config.subregionSize)
296 self.
warpCtrl = afwMath.WarpingControl(self.config.imageWarpMethod,
297 self.config.maskWarpMethod,
298 cacheSize=warpCache, interpLength=warpInterpLength)
299 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
300 self.config.dcrNumSubfilters,
301 filterInfo=filterInfo,
302 psf=templateCoadd.getPsf())
305 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
306 supplementaryData=None):
307 """Assemble the coadd. 309 Requires additional inputs Struct ``supplementaryData`` to contain a 310 ``templateCoadd`` that serves as the model of the static sky. 312 Find artifacts and apply them to the warps' masks creating a list of 313 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 314 Then pass these alternative masks to the base class's assemble method. 316 Divide the ``templateCoadd`` evenly between each subfilter of a 317 ``DcrModel`` as the starting best estimate of the true wavelength- 318 dependent sky. Forward model the ``DcrModel`` using the known 319 chromatic effects in each subfilter and calculate a convergence metric 320 based on how well the modeled template matches the input warps. If 321 the convergence has not yet reached the desired threshold, then shift 322 and stack the residual images to build a new ``DcrModel``. Apply 323 conditioning to prevent oscillating solutions between iterations or 326 Once the ``DcrModel`` reaches convergence or the maximum number of 327 iterations has been reached, fill the metadata for each subfilter 328 image and make them proper ``coaddExposure``s. 332 skyInfo : `lsst.pipe.base.Struct` 333 Patch geometry information, from getSkyInfo 334 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 335 The data references to the input warped exposures. 336 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 337 The image scalars correct for the zero point of the exposures. 338 weightList : `list` of `float` 339 The weight to give each input exposure in the coadd 340 supplementaryData : `lsst.pipe.base.Struct` 341 Result struct returned by ``makeSupplementaryData`` with components: 343 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 347 result : `lsst.pipe.base.Struct` 348 Result struct with components: 350 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 351 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 352 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 353 - ``dcrNImages``: `list` of exposure count images for each subfilter 355 templateCoadd = supplementaryData.templateCoadd
356 baseMask = templateCoadd.mask.clone()
359 baseVariance = templateCoadd.variance.clone()
360 baseVariance /= self.config.dcrNumSubfilters
361 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
363 templateCoadd.setMask(baseMask)
364 badMaskPlanes = self.config.badMaskPlanes[:]
365 badMaskPlanes.append(
"CLIPPED")
366 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
369 dcrModels = self.
prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
370 if self.config.doNImage:
372 tempExpRefList, spanSetMaskList, stats.ctrl)
373 nImage = afwImage.ImageU(skyInfo.bbox)
377 for dcrNImage
in dcrNImages:
382 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
383 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
384 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
386 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
389 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
390 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
391 dcrBBox = afwGeom.Box2I(subBBox)
393 dcrBBox.clip(dcrModels.bbox)
394 if self.config.useModelWeights:
399 imageScalerList, weightList, spanSetMaskList,
401 self.log.info(
"Initial convergence : %s", convergenceMetric)
402 convergenceList = [convergenceMetric]
404 convergenceCheck = 1.
405 subfilterVariance =
None 406 while (convergenceCheck > self.config.convergenceThreshold
or 407 modelIter < self.config.minNumIter):
410 weightList, spanSetMaskList, stats.flags, stats.ctrl,
411 convergenceMetric, baseMask, subfilterVariance, gain,
413 if self.config.useConvergence:
415 imageScalerList, weightList,
418 if convergenceMetric == 0:
419 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 420 "most likely due to there being no valid data in the region.",
421 skyInfo.patchInfo.getIndex(), subIter)
423 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
424 if convergenceCheck < 0:
425 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 426 "iterations or desired convergence improvement of %s." 428 skyInfo.patchInfo.getIndex(), subIter,
429 self.config.convergenceThreshold, convergenceCheck)
431 convergenceList.append(convergenceMetric)
432 if modelIter > self.config.maxNumIter:
433 if self.config.useConvergence:
434 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 435 "before reaching desired convergence improvement of %s." 436 " Final convergence improvement: %s",
437 skyInfo.patchInfo.getIndex(), subIter,
438 self.config.convergenceThreshold, convergenceCheck)
441 if self.config.useConvergence:
442 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
443 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
446 if self.config.useConvergence:
447 self.log.info(
"Coadd patch %s subregion %s finished with " 448 "convergence metric %s after %s iterations",
449 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
451 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
452 skyInfo.patchInfo.getIndex(), subIter, modelIter)
453 if self.config.useConvergence
and convergenceMetric > 0:
454 self.log.info(
"Final convergence improvement was %.4f%% overall",
455 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
457 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
458 calibration=self.scaleZeroPoint.getCalib(),
459 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
461 variance=baseVariance)
463 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
464 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
466 def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
467 """Calculate the number of exposures contributing to each subfilter. 471 dcrModels : `lsst.pipe.tasks.DcrModel` 472 Best fit model of the true sky after correcting chromatic effects. 473 bbox : `lsst.afw.geom.box.Box2I` 474 Bounding box of the patch to coadd. 475 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 476 The data references to the input warped exposures. 477 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 478 Each element is dict with keys = mask plane name to add the spans to 479 statsCtrl : `lsst.afw.math.StatisticsControl` 480 Statistics control object for coadd 484 dcrNImages : `list` of `lsst.afw.image.ImageU` 485 List of exposure count images for each subfilter 487 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
489 for tempExpRef, altMaskSpans
in zip(tempExpRefList, spanSetMaskList):
490 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
491 visitInfo = exposure.getInfo().getVisitInfo()
492 wcs = exposure.getInfo().getWcs()
494 if altMaskSpans
is not None:
496 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
497 for dcr, dcrNImage
in zip(dcrShift, dcrNImages):
498 shiftedImage = applyDcr(exposure.maskedImage, dcr, self.
warpCtrl, useInverse=
True)
499 dcrNImage.array[shiftedImage.mask.array & statsCtrl.getAndMask() == 0] += 1
502 def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList,
503 spanSetMaskList, statsFlags, statsCtrl, convergenceMetric,
504 baseMask, subfilterVariance, gain, modelWeights):
505 """Assemble the DCR coadd for a sub-region. 507 Build a DCR-matched template for each input exposure, then shift the 508 residuals according to the DCR in each subfilter. 509 Stack the shifted residuals and apply them as a correction to the 510 solution from the previous iteration. 511 Restrict the new model solutions from varying by more than a factor of 512 `modelClampFactor` from the last solution, and additionally restrict the 513 individual subfilter models from varying by more than a factor of 514 `frequencyClampFactor` from their average. 515 Finally, mitigate potentially oscillating solutions by averaging the new 516 solution with the solution from the previous iteration, weighted by 517 their convergence metric. 521 dcrModels : `lsst.pipe.tasks.DcrModel` 522 Best fit model of the true sky after correcting chromatic effects. 523 bbox : `lsst.afw.geom.box.Box2I` 524 Bounding box of the subregion to coadd. 525 dcrBBox :`lsst.afw.geom.box.Box2I` 526 Sub-region of the coadd which includes a buffer to allow for DCR. 527 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 528 The data references to the input warped exposures. 529 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 530 The image scalars correct for the zero point of the exposures. 531 weightList : `list` of `float` 532 The weight to give each input exposure in the coadd 533 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 534 Each element is dict with keys = mask plane name to add the spans to 535 statsFlags : `lsst.afw.math.Property` 536 Statistics settings for coaddition. 537 statsCtrl : `lsst.afw.math.StatisticsControl` 538 Statistics control object for coadd 539 convergenceMetric : `float` 540 Quality of fit metric for the matched templates of the input images. 541 baseMask : `lsst.afw.image.Mask` 542 Mask of the initial template coadd. 543 subfilterVariance : `list` of `numpy.ndarray` 544 The variance of each coadded subfilter image. 545 gain : `float`, optional 546 Relative weight to give the new solution when updating the model. 547 modelWeights : `numpy.ndarray` or `float` 548 A 2D array of weight values that tapers smoothly to zero away from detected sources. 549 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 552 residualGeneratorList = []
554 for tempExpRef, imageScaler, altMaskSpans
in zip(tempExpRefList, imageScalerList, spanSetMaskList):
555 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=dcrBBox)
556 visitInfo = exposure.getInfo().getVisitInfo()
557 wcs = exposure.getInfo().getWcs()
558 maskedImage = exposure.maskedImage
559 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl, visitInfo=visitInfo,
560 bbox=dcrBBox, wcs=wcs, mask=baseMask,
561 splitSubfilters=self.config.splitSubfilters)
562 imageScaler.scaleMaskedImage(maskedImage)
563 if altMaskSpans
is not None:
566 if self.config.removeMaskPlanes:
568 maskedImage -= templateImage
569 maskedImage.image.array *= modelWeights
570 residualGeneratorList.append(self.
dcrResiduals(maskedImage, visitInfo, dcrBBox, wcs,
574 statsFlags, statsCtrl, weightList,
575 mask=baseMask, gain=gain)
576 dcrModels.assign(dcrSubModelOut, bbox)
579 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 583 residual : `lsst.afw.image.MaskedImageF` 584 The residual masked image for one exposure, 585 after subtracting the matched template 586 visitInfo : `lsst.afw.image.VisitInfo` 587 Metadata for the exposure. 588 bbox : `lsst.afw.geom.box.Box2I` 589 Sub-region of the coadd 590 wcs : `lsst.afw.geom.SkyWcs` 591 Coordinate system definition (wcs) for the exposure. 592 filterInfo : `lsst.afw.image.Filter` 593 The filter definition, set in the current instruments' obs package. 594 Required for any calculation of DCR, including making matched templates. 598 residualImage : `lsst.afw.image.maskedImageF` 599 The residual image for the next subfilter, shifted for DCR. 601 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
603 yield applyDcr(residual, dcr, self.
warpCtrl, bbox=bbox, useInverse=
True)
606 statsFlags, statsCtrl, weightList,
608 """Calculate a new DcrModel from a set of image residuals. 612 dcrModels : `lsst.pipe.tasks.DcrModel` 613 Current model of the true sky after correcting chromatic effects. 614 residualGeneratorList : `generator` of `lsst.afw.image.maskedImageF` 615 The residual image for the next subfilter, shifted for DCR. 616 bbox : `lsst.afw.geom.box.Box2I` 617 Sub-region of the coadd 618 statsFlags : `lsst.afw.math.Property` 619 Statistics settings for coaddition. 620 statsCtrl : `lsst.afw.math.StatisticsControl` 621 Statistics control object for coadd 622 weightList : `list` of `float` 623 The weight to give each input exposure in the coadd 624 mask : `lsst.afw.image.Mask` 625 Mask to use for each new model image. 627 Relative weight to give the new solution when updating the model. 631 dcrModel : `lsst.pipe.tasks.DcrModel` 632 New model of the true sky after correcting chromatic effects. 635 clipped = dcrModels.mask.getPlaneBitMask(
"CLIPPED")
637 for subfilter, model
in enumerate(dcrModels):
638 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
639 residual = afwMath.statisticsStack(residualsList, statsFlags, statsCtrl, weightList,
641 residual.setXY0(bbox.getBegin())
643 residual += model[bbox]
646 badPixels = ~np.isfinite(newModel.image.array)
649 newModel.setMask(mask[bbox])
650 newModel.image.array[badPixels] = model[bbox].image.array[badPixels]
651 if self.config.regularizeModelIterations > 0:
652 dcrModels.regularizeModelIter(subfilter, newModel, bbox,
653 self.config.regularizeModelIterations,
654 self.config.regularizationWidth)
655 newModelImages.append(newModel)
656 if self.config.regularizeModelFrequency > 0:
657 dcrModels.regularizeModelFreq(newModelImages, bbox,
658 self.config.regularizeModelFrequency,
659 self.config.regularizationWidth)
660 dcrModels.conditionDcrModel(newModelImages, bbox, gain=gain)
661 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf)
664 weightList, spanSetMaskList, statsCtrl):
665 """Calculate a quality of fit metric for the matched templates. 669 dcrModels : `lsst.pipe.tasks.DcrModel` 670 Best fit model of the true sky after correcting chromatic effects. 671 bbox : `lsst.afw.geom.box.Box2I` 673 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 674 The data references to the input warped exposures. 675 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 676 The image scalars correct for the zero point of the exposures. 677 weightList : `list` of `float` 678 The weight to give each input exposure in the coadd 679 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 680 Each element is dict with keys = mask plane name to add the spans to 681 statsCtrl : `lsst.afw.math.StatisticsControl` 682 Statistics control object for coadd 686 convergenceMetric : `float` 687 Quality of fit metric for all input exposures, within the sub-region 689 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
691 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
697 zipIterables = zip(tempExpRefList, weightList, imageScalerList, spanSetMaskList)
698 for tempExpRef, expWeight, imageScaler, altMaskSpans
in zipIterables:
699 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
700 imageScaler.scaleMaskedImage(exposure.maskedImage)
702 altMaskSpans=altMaskSpans)
703 metric += singleMetric*expWeight
704 metricList[tempExpRef.dataId[
"visit"]] = singleMetric
706 self.log.info(
"Individual metrics:\n%s", metricList)
707 return 1.0
if weight == 0.0
else metric/weight
710 statsCtrl, altMaskSpans=None):
711 """Calculate a quality of fit metric for a single matched template. 715 dcrModels : `lsst.pipe.tasks.DcrModel` 716 Best fit model of the true sky after correcting chromatic effects. 717 exposure : `lsst.afw.image.ExposureF` 718 The input warped exposure to evaluate. 719 significanceImage : `numpy.ndarray` 720 Array of weights for each pixel corresponding to its significance 721 for the convergence calculation. 722 statsCtrl : `lsst.afw.math.StatisticsControl` 723 Statistics control object for coadd 724 altMaskSpans : `dict` containing spanSet lists, or None 725 The keys of the `dict` equal the mask plane name to add the spans to 729 convergenceMetric : `float` 730 Quality of fit metric for one exposure, within the sub-region. 732 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
733 templateImage = dcrModels.buildMatchedTemplate(warpCtrl=self.
warpCtrl,
734 visitInfo=exposure.getInfo().getVisitInfo(),
735 bbox=exposure.getBBox(),
736 wcs=exposure.getInfo().getWcs())
737 diffVals = np.abs(exposure.image.array - templateImage.image.array)*significanceImage
738 refVals = np.abs(templateImage.image.array)*significanceImage
740 finitePixels = np.isfinite(diffVals)
741 if altMaskSpans
is not None:
743 goodMaskPixels = exposure.mask.array & statsCtrl.getAndMask() == 0
744 convergeMaskPixels = exposure.mask.array & convergeMask > 0
745 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
746 if np.sum(usePixels) == 0:
749 diffUse = diffVals[usePixels]
750 refUse = refVals[usePixels]
751 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
755 """Add a list of sub-band coadds together. 759 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 760 A list of coadd exposures, each exposure containing 761 the model for one subfilter. 765 coaddExposure : `lsst.afw.image.ExposureF` 766 A single coadd exposure that is the sum of the sub-bands. 768 coaddExposure = dcrCoadds[0].clone()
769 for coadd
in dcrCoadds[1:]:
770 coaddExposure.maskedImage += coadd.maskedImage
773 def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
774 mask=None, variance=None):
775 """Create a list of coadd exposures from a list of masked images. 779 dcrModels : `lsst.pipe.tasks.DcrModel` 780 Best fit model of the true sky after correcting chromatic effects. 781 skyInfo : `lsst.pipe.base.Struct` 782 Patch geometry information, from getSkyInfo 783 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 784 The data references to the input warped exposures. 785 weightList : `list` of `float` 786 The weight to give each input exposure in the coadd 787 calibration : `lsst.afw.Image.Calib`, optional 788 Scale factor to set the photometric zero point of an exposure. 789 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 790 A record of the observations that are included in the coadd. 791 mask : `lsst.afw.image.Mask`, optional 792 Optional mask to override the values in the final coadd. 793 variance : `lsst.afw.image.Image`, optional 794 Optional variance plane to override the values in the final coadd. 798 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 799 A list of coadd exposures, each exposure containing 800 the model for one subfilter. 803 for model
in dcrModels:
804 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
805 if calibration
is not None:
806 coaddExposure.setCalib(calibration)
807 if coaddInputs
is not None:
808 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
811 coaddUtils.setCoaddEdgeBits(model[skyInfo.bbox].mask, model[skyInfo.bbox].variance)
812 coaddExposure.setMaskedImage(model[skyInfo.bbox])
814 coaddExposure.setMask(mask)
815 if variance
is not None:
816 coaddExposure.setVariance(variance)
817 dcrCoadds.append(coaddExposure)
821 """Calculate the gain to use for the current iteration. 823 After calculating a new DcrModel, each value is averaged with the 824 value in the corresponding pixel from the previous iteration. This 825 reduces oscillating solutions that iterative techniques are plagued by, 826 and speeds convergence. By far the biggest changes to the model 827 happen in the first couple iterations, so we can also use a more 828 aggressive gain later when the model is changing slowly. 832 convergenceList : `list` of `float` 833 The quality of fit metric from each previous iteration. 834 gainList : `list` of `float` 835 The gains used in each previous iteration: appended with the new 837 Gains are numbers between ``self.config.baseGain`` and 1. 842 Relative weight to give the new solution when updating the model. 843 A value of 1.0 gives equal weight to both solutions. 848 If ``len(convergenceList) != len(gainList)+1``. 850 nIter = len(convergenceList)
851 if nIter != len(gainList) + 1:
852 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 853 % (len(convergenceList), len(gainList)))
855 if self.config.baseGain
is None:
858 baseGain = 1./(self.config.dcrNumSubfilters - 1)
860 baseGain = self.config.baseGain
862 if self.config.useProgressiveGain
and nIter > 2:
870 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
871 for i
in range(nIter - 1)]
874 estFinalConv = np.array(estFinalConv)
875 estFinalConv[estFinalConv < 0] = 0
877 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
878 lastGain = gainList[-1]
879 lastConv = convergenceList[-2]
880 newConv = convergenceList[-1]
885 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
891 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
892 newGain = 1 - abs(delta)
894 newGain = (newGain + lastGain)/2.
895 gain = max(baseGain, newGain)
898 gainList.append(gain)
902 """Build an array that smoothly tapers to 0 away from detected sources. 906 dcrModels : `lsst.pipe.tasks.DcrModel` 907 Best fit model of the true sky after correcting chromatic effects. 908 dcrBBox : `lsst.afw.geom.box.Box2I` 909 Sub-region of the coadd which includes a buffer to allow for DCR. 913 weights : `numpy.ndarray` or `float` 914 A 2D array of weight values that tapers smoothly to zero away from detected sources. 915 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 920 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 922 if self.config.modelWeightsWidth < 0:
923 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
924 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
925 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
926 weights = np.zeros_like(dcrModels[0][dcrBBox].image.array)
927 weights[convergeMaskPixels] = 1.
928 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
929 weights /= np.max(weights)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def removeMaskPlanes(self, maskedImage)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl, altMaskSpans=None)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def calculateConvergence(self, dcrModels, bbox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsCtrl)
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, bbox, wcs, filterInfo)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def setRejectedMaskMapping(statsCtrl)
def newModelFromResidual(self, dcrModels, residualGeneratorList, bbox, statsFlags, statsCtrl, weightList, mask, gain)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, bbox, dcrBBox, tempExpRefList, imageScalerList, weightList, spanSetMaskList, statsFlags, statsCtrl, convergenceMetric, baseMask, subfilterVariance, gain, modelWeights)
def _subBBoxIter(bbox, subregionSize)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)