25 from scipy
import ndimage
32 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
34 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
38 dcrNumSubfilters = pexConfig.Field(
40 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
43 maxNumIter = pexConfig.Field(
46 doc=
"Maximum number of iterations of forward modeling.",
49 minNumIter = pexConfig.Field(
52 doc=
"Minimum number of iterations of forward modeling.",
55 convergenceThreshold = pexConfig.Field(
57 doc=
"Target relative change in convergence between iterations of forward modeling.",
60 useConvergence = pexConfig.Field(
62 doc=
"Use convergence test as a forward modeling end condition?" 63 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
66 baseGain = pexConfig.Field(
69 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 70 "A value of 1.0 gives equal weight to both solutions." 71 "Small values imply slower convergence of the solution, but can " 72 "help prevent overshooting and failures in the fit." 73 "If ``baseGain`` is None, a conservative gain " 74 "will be calculated from the number of subfilters. ",
77 useProgressiveGain = pexConfig.Field(
79 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 80 "When calculating the next gain, we use up to 5 previous gains and convergence values." 81 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
84 doAirmassWeight = pexConfig.Field(
86 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
89 modelWeightsWidth = pexConfig.Field(
91 doc=
"Width of the region around detected sources to include in the DcrModel.",
94 useModelWeights = pexConfig.Field(
96 doc=
"Width of the region around detected sources to include in the DcrModel.",
99 splitSubfilters = pexConfig.Field(
101 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 102 "Instead of at the midpoint",
105 regularizeModelIterations = pexConfig.Field(
107 doc=
"Maximum relative change of the model allowed between iterations." 108 "Set to zero to disable.",
111 regularizeModelFrequency = pexConfig.Field(
113 doc=
"Maximum relative change of the model allowed between subfilters." 114 "Set to zero to disable.",
117 convergenceMaskPlanes = pexConfig.ListField(
119 default=[
"DETECTED"],
120 doc=
"Mask planes to use to calculate convergence." 122 regularizationWidth = pexConfig.Field(
125 doc=
"Minimum radius of a region to include in regularization, in pixels." 127 imageInterpOrder = pexConfig.Field(
129 doc=
"The order of the spline interpolation used to shift the image plane.",
132 accelerateModel = pexConfig.Field(
134 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
139 CompareWarpAssembleCoaddConfig.setDefaults(self)
150 """Assemble DCR coadded images from a set of warps. 155 The number of pixels to grow each subregion by to allow for DCR. 159 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 160 Warps (also called coadded temporary exposures), including the effects of 161 Differential Chromatic Refraction (DCR). 162 For full details of the mathematics and algorithm, please see 163 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 165 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 166 each subfilter used in the iterative calculation. 167 It begins by dividing the bandpass-defining filter into N equal bandwidth 168 "subfilters", and divides the flux in each pixel from an initial coadd 169 equally into each as a "dcrModel". Because the airmass and parallactic 170 angle of each individual exposure is known, we can calculate the shift 171 relative to the center of the band in each subfilter due to DCR. For each 172 exposure we apply this shift as a linear transformation to the dcrModels 173 and stack the results to produce a DCR-matched exposure. The matched 174 exposures are subtracted from the input exposures to produce a set of 175 residual images, and these residuals are reverse shifted for each 176 exposures' subfilters and stacked. The shifted and stacked residuals are 177 added to the dcrModels to produce a new estimate of the flux in each pixel 178 within each subfilter. The dcrModels are solved for iteratively, which 179 continues until the solution from a new iteration improves by less than 180 a set percentage, or a maximum number of iterations is reached. 181 Two forms of regularization are employed to reduce unphysical results. 182 First, the new solution is averaged with the solution from the previous 183 iteration, which mitigates oscillating solutions where the model 184 overshoots with alternating very high and low values. 185 Second, a common degeneracy when the data have a limited range of airmass or 186 parallactic angle values is for one subfilter to be fit with very low or 187 negative values, while another subfilter is fit with very high values. This 188 typically appears in the form of holes next to sources in one subfilter, 189 and corresponding extended wings in another. Because each subfilter has 190 a narrow bandwidth we assume that physical sources that are above the noise 191 level will not vary in flux by more than a factor of `frequencyClampFactor` 192 between subfilters, and pixels that have flux deviations larger than that 193 factor will have the excess flux distributed evenly among all subfilters. 196 ConfigClass = DcrAssembleCoaddConfig
197 _DefaultName =
"dcrAssembleCoadd" 200 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
201 """Assemble a coadd from a set of warps. 203 Coadd a set of Warps. Compute weights to be applied to each Warp and 204 find scalings to match the photometric zeropoint to a reference Warp. 205 Assemble the Warps using run method. 206 Forward model chromatic effects across multiple subfilters, 207 and subtract from the input Warps to build sets of residuals. 208 Use the residuals to construct a new ``DcrModel`` for each subfilter, 209 and iterate until the model converges. 210 Interpolate over NaNs and optionally write the coadd to disk. 211 Return the coadded exposure. 215 dataRef : `lsst.daf.persistence.ButlerDataRef` 216 Data reference defining the patch for coaddition and the 218 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 219 List of data references to warps. Data to be coadded will be 220 selected from this list based on overlap with the patch defined by 225 results : `lsst.pipe.base.Struct` 226 The Struct contains the following fields: 228 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 229 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 230 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 231 - ``dcrNImages``: `list` of exposure count images for each subfilter 233 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
234 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
236 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
237 warpRefList=warpRefList)
240 self.log.warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
241 skyInfo.patchInfo.getIndex())
243 for subfilter
in range(self.config.dcrNumSubfilters):
245 if self.config.doWrite:
246 self.log.info(
"Persisting dcrCoadd")
247 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
248 numSubfilters=self.config.dcrNumSubfilters)
249 if self.config.doNImage
and results.dcrNImages
is not None:
250 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
251 numSubfilters=self.config.dcrNumSubfilters)
256 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 258 Sets the property ``bufferSize``. 262 templateCoadd : `lsst.afw.image.ExposureF` 263 The initial coadd exposure before accounting for DCR. 264 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 265 The data references to the input warped exposures. 266 weightList : `list` of `float` 267 The weight to give each input exposure in the coadd 268 Will be modified in place if ``doAirmassWeight`` is set. 272 dcrModels : `lsst.pipe.tasks.DcrModel` 273 Best fit model of the true sky after correcting chromatic effects. 278 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 280 filterInfo = templateCoadd.getFilter()
281 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
282 raise NotImplementedError(
"No minimum/maximum wavelength information found" 283 " in the filter definition! Please add lambdaMin and lambdaMax" 284 " to the Mapper class in your obs package.")
289 for visitNum, tempExpRef
in enumerate(tempExpRefList):
290 visitInfo = tempExpRef.get(tempExpName +
"_visitInfo")
291 airmass = visitInfo.getBoresightAirmass()
292 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
293 airmassList[tempExpRef.dataId[
"visit"]] = airmass
294 angleList[tempExpRef.dataId[
"visit"]] = parallacticAngle
295 if self.config.doAirmassWeight:
296 weightList[visitNum] *= airmass
297 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
298 filterInfo, self.config.dcrNumSubfilters))))
299 self.log.info(
"Selected airmasses:\n%s", airmassList)
300 self.log.info(
"Selected parallactic angles:\n%s", angleList)
302 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
303 self.config.dcrNumSubfilters,
304 filterInfo=filterInfo,
305 psf=templateCoadd.getPsf())
308 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
309 supplementaryData=None):
310 """Assemble the coadd. 312 Requires additional inputs Struct ``supplementaryData`` to contain a 313 ``templateCoadd`` that serves as the model of the static sky. 315 Find artifacts and apply them to the warps' masks creating a list of 316 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 317 Then pass these alternative masks to the base class's assemble method. 319 Divide the ``templateCoadd`` evenly between each subfilter of a 320 ``DcrModel`` as the starting best estimate of the true wavelength- 321 dependent sky. Forward model the ``DcrModel`` using the known 322 chromatic effects in each subfilter and calculate a convergence metric 323 based on how well the modeled template matches the input warps. If 324 the convergence has not yet reached the desired threshold, then shift 325 and stack the residual images to build a new ``DcrModel``. Apply 326 conditioning to prevent oscillating solutions between iterations or 329 Once the ``DcrModel`` reaches convergence or the maximum number of 330 iterations has been reached, fill the metadata for each subfilter 331 image and make them proper ``coaddExposure``s. 335 skyInfo : `lsst.pipe.base.Struct` 336 Patch geometry information, from getSkyInfo 337 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 338 The data references to the input warped exposures. 339 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 340 The image scalars correct for the zero point of the exposures. 341 weightList : `list` of `float` 342 The weight to give each input exposure in the coadd 343 supplementaryData : `lsst.pipe.base.Struct` 344 Result struct returned by ``makeSupplementaryData`` with components: 346 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 350 result : `lsst.pipe.base.Struct` 351 Result struct with components: 353 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 354 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 355 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 356 - ``dcrNImages``: `list` of exposure count images for each subfilter 358 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
359 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
360 templateCoadd = supplementaryData.templateCoadd
361 baseMask = templateCoadd.mask.clone()
364 baseVariance = templateCoadd.variance.clone()
365 baseVariance /= self.config.dcrNumSubfilters
366 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
368 templateCoadd.setMask(baseMask)
369 badMaskPlanes = self.config.badMaskPlanes[:]
374 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
377 dcrModels = self.
prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
378 if self.config.doNImage:
379 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, tempExpRefList,
380 spanSetMaskList, stats.ctrl)
381 nImage = afwImage.ImageU(skyInfo.bbox)
385 for dcrNImage
in dcrNImages:
390 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
391 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
392 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
394 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
397 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
398 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
399 dcrBBox = afwGeom.Box2I(subBBox)
401 dcrBBox.clip(dcrModels.bbox)
404 imageScalerList, spanSetMaskList)
406 tempExpRefList, weightList, stats.ctrl)
407 self.log.info(
"Initial convergence : %s", convergenceMetric)
408 convergenceList = [convergenceMetric]
410 convergenceCheck = 1.
411 refImage = templateCoadd.image
412 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
415 stats.ctrl, convergenceMetric, baseMask, gain,
416 modelWeights, refImage, dcrWeights)
417 if self.config.useConvergence:
419 tempExpRefList, weightList, stats.ctrl)
420 if convergenceMetric == 0:
421 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 422 "most likely due to there being no valid data in the region.",
423 skyInfo.patchInfo.getIndex(), subIter)
425 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
426 if (convergenceCheck < 0) & (modelIter > minNumIter):
427 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 428 "iterations or desired convergence improvement of %s." 430 skyInfo.patchInfo.getIndex(), subIter,
431 self.config.convergenceThreshold, convergenceCheck)
433 convergenceList.append(convergenceMetric)
434 if modelIter > maxNumIter:
435 if self.config.useConvergence:
436 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 437 "before reaching desired convergence improvement of %s." 438 " Final convergence improvement: %s",
439 skyInfo.patchInfo.getIndex(), subIter,
440 self.config.convergenceThreshold, convergenceCheck)
443 if self.config.useConvergence:
444 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
445 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
448 if self.config.useConvergence:
449 self.log.info(
"Coadd patch %s subregion %s finished with " 450 "convergence metric %s after %s iterations",
451 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
453 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
454 skyInfo.patchInfo.getIndex(), subIter, modelIter)
455 if self.config.useConvergence
and convergenceMetric > 0:
456 self.log.info(
"Final convergence improvement was %.4f%% overall",
457 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
459 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
460 calibration=self.scaleZeroPoint.getPhotoCalib(),
461 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
463 variance=baseVariance)
465 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
466 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
468 def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
469 """Calculate the number of exposures contributing to each subfilter. 473 dcrModels : `lsst.pipe.tasks.DcrModel` 474 Best fit model of the true sky after correcting chromatic effects. 475 bbox : `lsst.afw.geom.box.Box2I` 476 Bounding box of the patch to coadd. 477 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 478 The data references to the input warped exposures. 479 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 480 Each element of the `dict` contains the new mask plane name 481 (e.g. "CLIPPED and/or "NO_DATA") as the key, 482 and the list of SpanSets to apply to the mask. 483 statsCtrl : `lsst.afw.math.StatisticsControl` 484 Statistics control object for coadd 488 dcrNImages : `list` of `lsst.afw.image.ImageU` 489 List of exposure count images for each subfilter 490 dcrWeights : `list` of `lsst.afw.image.ImageF` 491 Per-pixel weights for each subfilter. 492 Equal to 1/(number of unmasked images contributing to each pixel). 494 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
495 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
497 for tempExpRef, altMaskSpans
in zip(tempExpRefList, spanSetMaskList):
498 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
499 visitInfo = exposure.getInfo().getVisitInfo()
500 wcs = exposure.getInfo().getWcs()
502 if altMaskSpans
is not None:
504 weightImage = np.zeros_like(exposure.image.array)
505 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
506 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
507 for dcr, dcrNImage, dcrWeight
in zip(dcrShift, dcrNImages, dcrWeights):
509 shiftedWeights = applyDcr(weightImage, dcr, useInverse=
True,
510 order=self.config.imageInterpOrder)
511 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
512 dcrWeight.array += shiftedWeights
514 weightsThreshold = 1.
515 goodPix = dcrWeights[0].array > weightsThreshold
516 for weights
in dcrWeights[1:]:
517 goodPix = (weights.array > weightsThreshold) & goodPix
518 for subfilter
in range(self.config.dcrNumSubfilters):
519 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
520 dcrWeights[subfilter].array[~goodPix] = 0.
521 dcrNImages[subfilter].array[~goodPix] = 0
522 return (dcrNImages, dcrWeights)
525 statsCtrl, convergenceMetric,
526 baseMask, gain, modelWeights, refImage, dcrWeights):
527 """Assemble the DCR coadd for a sub-region. 529 Build a DCR-matched template for each input exposure, then shift the 530 residuals according to the DCR in each subfilter. 531 Stack the shifted residuals and apply them as a correction to the 532 solution from the previous iteration. 533 Restrict the new model solutions from varying by more than a factor of 534 `modelClampFactor` from the last solution, and additionally restrict the 535 individual subfilter models from varying by more than a factor of 536 `frequencyClampFactor` from their average. 537 Finally, mitigate potentially oscillating solutions by averaging the new 538 solution with the solution from the previous iteration, weighted by 539 their convergence metric. 543 dcrModels : `lsst.pipe.tasks.DcrModel` 544 Best fit model of the true sky after correcting chromatic effects. 545 subExposures : `dict` of `lsst.afw.image.ExposureF` 546 The pre-loaded exposures for the current subregion. 547 bbox : `lsst.afw.geom.box.Box2I` 548 Bounding box of the subregion to coadd. 549 dcrBBox : `lsst.afw.geom.box.Box2I` 550 Sub-region of the coadd which includes a buffer to allow for DCR. 551 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 552 The data references to the input warped exposures. 553 statsCtrl : `lsst.afw.math.StatisticsControl` 554 Statistics control object for coadd 555 convergenceMetric : `float` 556 Quality of fit metric for the matched templates of the input images. 557 baseMask : `lsst.afw.image.Mask` 558 Mask of the initial template coadd. 559 gain : `float`, optional 560 Relative weight to give the new solution when updating the model. 561 modelWeights : `numpy.ndarray` or `float` 562 A 2D array of weight values that tapers smoothly to zero away from detected sources. 563 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 564 refImage : `lsst.afw.image.Image` 565 A reference image used to supply the default pixel values. 566 dcrWeights : `list` of `lsst.afw.image.Image` 567 Per-pixel weights for each subfilter. 568 Equal to 1/(number of unmasked images contributing to each pixel). 570 residualGeneratorList = []
572 for tempExpRef
in tempExpRefList:
573 exposure = subExposures[tempExpRef.dataId[
"visit"]]
574 visitInfo = exposure.getInfo().getVisitInfo()
575 wcs = exposure.getInfo().getWcs()
576 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
577 order=self.config.imageInterpOrder,
578 splitSubfilters=self.config.splitSubfilters,
579 amplifyModel=self.config.accelerateModel)
580 residual = exposure.image.array - templateImage.array
582 residual *= exposure.variance.array
586 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
588 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
589 mask=baseMask, gain=gain,
590 modelWeights=modelWeights,
592 dcrWeights=dcrWeights)
593 dcrModels.assign(dcrSubModelOut, bbox)
596 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 600 residual : `numpy.ndarray` 601 The residual masked image for one exposure, 602 after subtracting the matched template 603 visitInfo : `lsst.afw.image.VisitInfo` 604 Metadata for the exposure. 605 wcs : `lsst.afw.geom.SkyWcs` 606 Coordinate system definition (wcs) for the exposure. 607 filterInfo : `lsst.afw.image.Filter` 608 The filter definition, set in the current instruments' obs package. 609 Required for any calculation of DCR, including making matched templates. 613 residualImage : `numpy.ndarray` 614 The residual image for the next subfilter, shifted for DCR. 616 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
618 yield applyDcr(residual, dcr, useInverse=
True, order=self.config.imageInterpOrder)
621 mask, gain, modelWeights, refImage, dcrWeights):
622 """Calculate a new DcrModel from a set of image residuals. 626 dcrModels : `lsst.pipe.tasks.DcrModel` 627 Current model of the true sky after correcting chromatic effects. 628 residualGeneratorList : `generator` of `numpy.ndarray` 629 The residual image for the next subfilter, shifted for DCR. 630 dcrBBox : `lsst.afw.geom.box.Box2I` 631 Sub-region of the coadd which includes a buffer to allow for DCR. 632 statsCtrl : `lsst.afw.math.StatisticsControl` 633 Statistics control object for coadd 634 mask : `lsst.afw.image.Mask` 635 Mask to use for each new model image. 637 Relative weight to give the new solution when updating the model. 638 modelWeights : `numpy.ndarray` or `float` 639 A 2D array of weight values that tapers smoothly to zero away from detected sources. 640 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 641 refImage : `lsst.afw.image.Image` 642 A reference image used to supply the default pixel values. 643 dcrWeights : `list` of `lsst.afw.image.Image` 644 Per-pixel weights for each subfilter. 645 Equal to 1/(number of unmasked images contributing to each pixel). 649 dcrModel : `lsst.pipe.tasks.DcrModel` 650 New model of the true sky after correcting chromatic effects. 653 for subfilter, model
in enumerate(dcrModels):
654 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
655 residual = np.sum(residualsList, axis=0)
656 residual *= dcrWeights[subfilter][dcrBBox].array
658 newModel = model[dcrBBox].clone()
659 newModel.array += residual
661 badPixels = ~np.isfinite(newModel.array)
662 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
663 if self.config.regularizeModelIterations > 0:
664 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
665 self.config.regularizeModelIterations,
666 self.config.regularizationWidth)
667 newModelImages.append(newModel)
668 if self.config.regularizeModelFrequency > 0:
669 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
670 self.config.regularizeModelFrequency,
671 self.config.regularizationWidth,
673 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
675 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
676 dcrModels.mask, dcrModels.variance)
679 """Calculate a quality of fit metric for the matched templates. 683 dcrModels : `lsst.pipe.tasks.DcrModel` 684 Best fit model of the true sky after correcting chromatic effects. 685 subExposures : `dict` of `lsst.afw.image.ExposureF` 686 The pre-loaded exposures for the current subregion. 687 bbox : `lsst.afw.geom.box.Box2I` 689 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 690 The data references to the input warped exposures. 691 weightList : `list` of `float` 692 The weight to give each input exposure in the coadd 693 statsCtrl : `lsst.afw.math.StatisticsControl` 694 Statistics control object for coadd 698 convergenceMetric : `float` 699 Quality of fit metric for all input exposures, within the sub-region 701 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
703 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
705 if np.max(significanceImage) == 0:
706 significanceImage += 1.
710 for tempExpRef, expWeight
in zip(tempExpRefList, weightList):
711 exposure = subExposures[tempExpRef.dataId[
"visit"]][bbox]
713 metric += singleMetric
714 metricList[tempExpRef.dataId[
"visit"]] = singleMetric
716 self.log.info(
"Individual metrics:\n%s", metricList)
717 return 1.0
if weight == 0.0
else metric/weight
720 """Calculate a quality of fit metric for a single matched template. 724 dcrModels : `lsst.pipe.tasks.DcrModel` 725 Best fit model of the true sky after correcting chromatic effects. 726 exposure : `lsst.afw.image.ExposureF` 727 The input warped exposure to evaluate. 728 significanceImage : `numpy.ndarray` 729 Array of weights for each pixel corresponding to its significance 730 for the convergence calculation. 731 statsCtrl : `lsst.afw.math.StatisticsControl` 732 Statistics control object for coadd 736 convergenceMetric : `float` 737 Quality of fit metric for one exposure, within the sub-region. 739 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
740 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
741 order=self.config.imageInterpOrder,
742 splitSubfilters=self.config.splitSubfilters,
743 amplifyModel=self.config.accelerateModel)
744 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
745 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
747 finitePixels = np.isfinite(diffVals)
748 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
749 convergeMaskPixels = exposure.mask.array & convergeMask > 0
750 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
751 if np.sum(usePixels) == 0:
754 diffUse = diffVals[usePixels]
755 refUse = refVals[usePixels]
756 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
760 """Add a list of sub-band coadds together. 764 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 765 A list of coadd exposures, each exposure containing 766 the model for one subfilter. 770 coaddExposure : `lsst.afw.image.ExposureF` 771 A single coadd exposure that is the sum of the sub-bands. 773 coaddExposure = dcrCoadds[0].clone()
774 for coadd
in dcrCoadds[1:]:
775 coaddExposure.maskedImage += coadd.maskedImage
778 def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
779 mask=None, variance=None):
780 """Create a list of coadd exposures from a list of masked images. 784 dcrModels : `lsst.pipe.tasks.DcrModel` 785 Best fit model of the true sky after correcting chromatic effects. 786 skyInfo : `lsst.pipe.base.Struct` 787 Patch geometry information, from getSkyInfo 788 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 789 The data references to the input warped exposures. 790 weightList : `list` of `float` 791 The weight to give each input exposure in the coadd 792 calibration : `lsst.afw.Image.PhotoCalib`, optional 793 Scale factor to set the photometric calibration of an exposure. 794 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 795 A record of the observations that are included in the coadd. 796 mask : `lsst.afw.image.Mask`, optional 797 Optional mask to override the values in the final coadd. 798 variance : `lsst.afw.image.Image`, optional 799 Optional variance plane to override the values in the final coadd. 803 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 804 A list of coadd exposures, each exposure containing 805 the model for one subfilter. 808 refModel = dcrModels.getReferenceImage()
809 for model
in dcrModels:
810 if self.config.accelerateModel > 1:
811 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
812 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
813 if calibration
is not None:
814 coaddExposure.setPhotoCalib(calibration)
815 if coaddInputs
is not None:
816 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
819 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
820 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
821 maskedImage.image = model
822 maskedImage.mask = dcrModels.mask
823 maskedImage.variance = dcrModels.variance
824 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
826 coaddExposure.setMask(mask)
827 if variance
is not None:
828 coaddExposure.setVariance(variance)
829 dcrCoadds.append(coaddExposure)
833 """Calculate the gain to use for the current iteration. 835 After calculating a new DcrModel, each value is averaged with the 836 value in the corresponding pixel from the previous iteration. This 837 reduces oscillating solutions that iterative techniques are plagued by, 838 and speeds convergence. By far the biggest changes to the model 839 happen in the first couple iterations, so we can also use a more 840 aggressive gain later when the model is changing slowly. 844 convergenceList : `list` of `float` 845 The quality of fit metric from each previous iteration. 846 gainList : `list` of `float` 847 The gains used in each previous iteration: appended with the new 849 Gains are numbers between ``self.config.baseGain`` and 1. 854 Relative weight to give the new solution when updating the model. 855 A value of 1.0 gives equal weight to both solutions. 860 If ``len(convergenceList) != len(gainList)+1``. 862 nIter = len(convergenceList)
863 if nIter != len(gainList) + 1:
864 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 865 % (len(convergenceList), len(gainList)))
867 if self.config.baseGain
is None:
870 baseGain = 1./(self.config.dcrNumSubfilters - 1)
872 baseGain = self.config.baseGain
874 if self.config.useProgressiveGain
and nIter > 2:
882 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
883 for i
in range(nIter - 1)]
886 estFinalConv = np.array(estFinalConv)
887 estFinalConv[estFinalConv < 0] = 0
889 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
890 lastGain = gainList[-1]
891 lastConv = convergenceList[-2]
892 newConv = convergenceList[-1]
897 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
903 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
904 newGain = 1 - abs(delta)
906 newGain = (newGain + lastGain)/2.
907 gain = max(baseGain, newGain)
910 gainList.append(gain)
914 """Build an array that smoothly tapers to 0 away from detected sources. 918 dcrModels : `lsst.pipe.tasks.DcrModel` 919 Best fit model of the true sky after correcting chromatic effects. 920 dcrBBox : `lsst.afw.geom.box.Box2I` 921 Sub-region of the coadd which includes a buffer to allow for DCR. 925 weights : `numpy.ndarray` or `float` 926 A 2D array of weight values that tapers smoothly to zero away from detected sources. 927 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 932 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 934 if not self.config.useModelWeights:
936 if self.config.modelWeightsWidth < 0:
937 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
938 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
939 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
940 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
941 weights[convergeMaskPixels] = 1.
942 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
943 weights /= np.max(weights)
947 """Smoothly replace model pixel values with those from a 948 reference at locations away from detected sources. 952 modelImages : `list` of `lsst.afw.image.Image` 953 The new DCR model images from the current iteration. 954 The values will be modified in place. 955 refImage : `lsst.afw.image.MaskedImage` 956 A reference image used to supply the default pixel values. 957 modelWeights : `numpy.ndarray` or `float` 958 A 2D array of weight values that tapers smoothly to zero away from detected sources. 959 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 961 if self.config.useModelWeights:
962 for model
in modelImages:
963 model.array *= modelWeights
964 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
966 def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList):
967 """Pre-load sub-regions of a list of exposures. 971 bbox : `lsst.afw.geom.box.Box2I` 973 statsCtrl : `lsst.afw.math.StatisticsControl` 974 Statistics control object for coadd 975 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 976 The data references to the input warped exposures. 977 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 978 The image scalars correct for the zero point of the exposures. 979 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 980 Each element is dict with keys = mask plane name to add the spans to 984 subExposures : `dict` 985 The `dict` keys are the visit IDs, 986 and the values are `lsst.afw.image.ExposureF` 987 The pre-loaded exposures for the current subregion. 988 The variance plane contains weights, and not the variance 991 zipIterables = zip(tempExpRefList, imageScalerList, spanSetMaskList)
993 for tempExpRef, imageScaler, altMaskSpans
in zipIterables:
994 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
995 if altMaskSpans
is not None:
997 imageScaler.scaleMaskedImage(exposure.maskedImage)
999 exposure.variance.array[:, :] = 0.
1001 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1004 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1005 subExposures[tempExpRef.dataId[
"visit"]] = exposure
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, tempExpRefList, weightList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, tempExpRefList, statsCtrl, convergenceMetric, baseMask, gain, modelWeights, refImage, dcrWeights)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, mask, gain, modelWeights, refImage, dcrWeights)
def _subBBoxIter(bbox, subregionSize)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)
def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList)