25 from scipy
import ndimage
32 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
34 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
38 dcrNumSubfilters = pexConfig.Field(
40 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
43 maxNumIter = pexConfig.Field(
46 doc=
"Maximum number of iterations of forward modeling.",
49 minNumIter = pexConfig.Field(
52 doc=
"Minimum number of iterations of forward modeling.",
55 convergenceThreshold = pexConfig.Field(
57 doc=
"Target relative change in convergence between iterations of forward modeling.",
60 useConvergence = pexConfig.Field(
62 doc=
"Use convergence test as a forward modeling end condition?" 63 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
66 baseGain = pexConfig.Field(
69 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 70 "A value of 1.0 gives equal weight to both solutions." 71 "Small values imply slower convergence of the solution, but can " 72 "help prevent overshooting and failures in the fit." 73 "If ``baseGain`` is None, a conservative gain " 74 "will be calculated from the number of subfilters. ",
77 useProgressiveGain = pexConfig.Field(
79 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 80 "When calculating the next gain, we use up to 5 previous gains and convergence values." 81 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
84 doAirmassWeight = pexConfig.Field(
86 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
89 modelWeightsWidth = pexConfig.Field(
91 doc=
"Width of the region around detected sources to include in the DcrModel.",
94 useModelWeights = pexConfig.Field(
96 doc=
"Width of the region around detected sources to include in the DcrModel.",
99 splitSubfilters = pexConfig.Field(
101 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 102 "Instead of at the midpoint",
105 regularizeModelIterations = pexConfig.Field(
107 doc=
"Maximum relative change of the model allowed between iterations." 108 "Set to zero to disable.",
111 regularizeModelFrequency = pexConfig.Field(
113 doc=
"Maximum relative change of the model allowed between subfilters." 114 "Set to zero to disable.",
117 convergenceMaskPlanes = pexConfig.ListField(
119 default=[
"DETECTED"],
120 doc=
"Mask planes to use to calculate convergence." 122 regularizationWidth = pexConfig.Field(
125 doc=
"Minimum radius of a region to include in regularization, in pixels." 127 imageInterpOrder = pexConfig.Field(
129 doc=
"The order of the spline interpolation used to shift the image plane.",
134 CompareWarpAssembleCoaddConfig.setDefaults(self)
145 """Assemble DCR coadded images from a set of warps. 150 The number of pixels to grow each subregion by to allow for DCR. 154 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 155 Warps (also called coadded temporary exposures), including the effects of 156 Differential Chromatic Refraction (DCR). 157 For full details of the mathematics and algorithm, please see 158 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 160 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 161 each subfilter used in the iterative calculation. 162 It begins by dividing the bandpass-defining filter into N equal bandwidth 163 "subfilters", and divides the flux in each pixel from an initial coadd 164 equally into each as a "dcrModel". Because the airmass and parallactic 165 angle of each individual exposure is known, we can calculate the shift 166 relative to the center of the band in each subfilter due to DCR. For each 167 exposure we apply this shift as a linear transformation to the dcrModels 168 and stack the results to produce a DCR-matched exposure. The matched 169 exposures are subtracted from the input exposures to produce a set of 170 residual images, and these residuals are reverse shifted for each 171 exposures' subfilters and stacked. The shifted and stacked residuals are 172 added to the dcrModels to produce a new estimate of the flux in each pixel 173 within each subfilter. The dcrModels are solved for iteratively, which 174 continues until the solution from a new iteration improves by less than 175 a set percentage, or a maximum number of iterations is reached. 176 Two forms of regularization are employed to reduce unphysical results. 177 First, the new solution is averaged with the solution from the previous 178 iteration, which mitigates oscillating solutions where the model 179 overshoots with alternating very high and low values. 180 Second, a common degeneracy when the data have a limited range of airmass or 181 parallactic angle values is for one subfilter to be fit with very low or 182 negative values, while another subfilter is fit with very high values. This 183 typically appears in the form of holes next to sources in one subfilter, 184 and corresponding extended wings in another. Because each subfilter has 185 a narrow bandwidth we assume that physical sources that are above the noise 186 level will not vary in flux by more than a factor of `frequencyClampFactor` 187 between subfilters, and pixels that have flux deviations larger than that 188 factor will have the excess flux distributed evenly among all subfilters. 191 ConfigClass = DcrAssembleCoaddConfig
192 _DefaultName =
"dcrAssembleCoadd" 195 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
196 """Assemble a coadd from a set of warps. 198 Coadd a set of Warps. Compute weights to be applied to each Warp and 199 find scalings to match the photometric zeropoint to a reference Warp. 200 Assemble the Warps using run method. 201 Forward model chromatic effects across multiple subfilters, 202 and subtract from the input Warps to build sets of residuals. 203 Use the residuals to construct a new ``DcrModel`` for each subfilter, 204 and iterate until the model converges. 205 Interpolate over NaNs and optionally write the coadd to disk. 206 Return the coadded exposure. 210 dataRef : `lsst.daf.persistence.ButlerDataRef` 211 Data reference defining the patch for coaddition and the 213 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 214 List of data references to warps. Data to be coadded will be 215 selected from this list based on overlap with the patch defined by 220 results : `lsst.pipe.base.Struct` 221 The Struct contains the following fields: 223 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 224 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 225 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 226 - ``dcrNImages``: `list` of exposure count images for each subfilter 228 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
229 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
231 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
232 warpRefList=warpRefList)
235 self.log.warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
236 skyInfo.patchInfo.getIndex())
238 for subfilter
in range(self.config.dcrNumSubfilters):
240 if self.config.doWrite:
241 self.log.info(
"Persisting dcrCoadd")
242 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
243 numSubfilters=self.config.dcrNumSubfilters)
244 if self.config.doNImage
and results.dcrNImages
is not None:
245 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
246 numSubfilters=self.config.dcrNumSubfilters)
251 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 253 Sets the property ``bufferSize``. 257 templateCoadd : `lsst.afw.image.ExposureF` 258 The initial coadd exposure before accounting for DCR. 259 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 260 The data references to the input warped exposures. 261 weightList : `list` of `float` 262 The weight to give each input exposure in the coadd 263 Will be modified in place if ``doAirmassWeight`` is set. 267 dcrModels : `lsst.pipe.tasks.DcrModel` 268 Best fit model of the true sky after correcting chromatic effects. 273 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 275 filterInfo = templateCoadd.getFilter()
276 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
277 raise NotImplementedError(
"No minimum/maximum wavelength information found" 278 " in the filter definition! Please add lambdaMin and lambdaMax" 279 " to the Mapper class in your obs package.")
284 for visitNum, tempExpRef
in enumerate(tempExpRefList):
285 visitInfo = tempExpRef.get(tempExpName +
"_visitInfo")
286 airmass = visitInfo.getBoresightAirmass()
287 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
288 airmassList[tempExpRef.dataId[
"visit"]] = airmass
289 angleList[tempExpRef.dataId[
"visit"]] = parallacticAngle
290 if self.config.doAirmassWeight:
291 weightList[visitNum] *= airmass
292 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
293 filterInfo, self.config.dcrNumSubfilters))))
294 self.log.info(
"Selected airmasses:\n%s", airmassList)
295 self.log.info(
"Selected parallactic angles:\n%s", angleList)
297 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
298 self.config.dcrNumSubfilters,
299 filterInfo=filterInfo,
300 psf=templateCoadd.getPsf())
303 def run(self, skyInfo, tempExpRefList, imageScalerList, weightList,
304 supplementaryData=None):
305 """Assemble the coadd. 307 Requires additional inputs Struct ``supplementaryData`` to contain a 308 ``templateCoadd`` that serves as the model of the static sky. 310 Find artifacts and apply them to the warps' masks creating a list of 311 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 312 Then pass these alternative masks to the base class's assemble method. 314 Divide the ``templateCoadd`` evenly between each subfilter of a 315 ``DcrModel`` as the starting best estimate of the true wavelength- 316 dependent sky. Forward model the ``DcrModel`` using the known 317 chromatic effects in each subfilter and calculate a convergence metric 318 based on how well the modeled template matches the input warps. If 319 the convergence has not yet reached the desired threshold, then shift 320 and stack the residual images to build a new ``DcrModel``. Apply 321 conditioning to prevent oscillating solutions between iterations or 324 Once the ``DcrModel`` reaches convergence or the maximum number of 325 iterations has been reached, fill the metadata for each subfilter 326 image and make them proper ``coaddExposure``s. 330 skyInfo : `lsst.pipe.base.Struct` 331 Patch geometry information, from getSkyInfo 332 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 333 The data references to the input warped exposures. 334 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 335 The image scalars correct for the zero point of the exposures. 336 weightList : `list` of `float` 337 The weight to give each input exposure in the coadd 338 supplementaryData : `lsst.pipe.base.Struct` 339 Result struct returned by ``makeSupplementaryData`` with components: 341 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 345 result : `lsst.pipe.base.Struct` 346 Result struct with components: 348 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 349 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 350 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 351 - ``dcrNImages``: `list` of exposure count images for each subfilter 353 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
354 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
355 templateCoadd = supplementaryData.templateCoadd
356 baseMask = templateCoadd.mask.clone()
359 baseVariance = templateCoadd.variance.clone()
360 baseVariance /= self.config.dcrNumSubfilters
361 spanSetMaskList = self.
findArtifacts(templateCoadd, tempExpRefList, imageScalerList)
363 templateCoadd.setMask(baseMask)
364 badMaskPlanes = self.config.badMaskPlanes[:]
369 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
372 dcrModels = self.
prepareDcrInputs(templateCoadd, tempExpRefList, weightList)
373 if self.config.doNImage:
374 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, tempExpRefList,
375 spanSetMaskList, stats.ctrl)
376 nImage = afwImage.ImageU(skyInfo.bbox)
380 for dcrNImage
in dcrNImages:
385 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
386 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
387 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
389 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
392 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
393 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
394 dcrBBox = afwGeom.Box2I(subBBox)
396 dcrBBox.clip(dcrModels.bbox)
399 imageScalerList, spanSetMaskList)
401 tempExpRefList, weightList, stats.ctrl)
402 self.log.info(
"Initial convergence : %s", convergenceMetric)
403 convergenceList = [convergenceMetric]
405 convergenceCheck = 1.
406 refImage = templateCoadd.image
407 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
410 stats.ctrl, convergenceMetric, baseMask, gain,
411 modelWeights, refImage, dcrWeights)
412 if self.config.useConvergence:
414 tempExpRefList, weightList, stats.ctrl)
415 if convergenceMetric == 0:
416 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 417 "most likely due to there being no valid data in the region.",
418 skyInfo.patchInfo.getIndex(), subIter)
420 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
421 if (convergenceCheck < 0) & (modelIter > minNumIter):
422 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 423 "iterations or desired convergence improvement of %s." 425 skyInfo.patchInfo.getIndex(), subIter,
426 self.config.convergenceThreshold, convergenceCheck)
428 convergenceList.append(convergenceMetric)
429 if modelIter > maxNumIter:
430 if self.config.useConvergence:
431 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 432 "before reaching desired convergence improvement of %s." 433 " Final convergence improvement: %s",
434 skyInfo.patchInfo.getIndex(), subIter,
435 self.config.convergenceThreshold, convergenceCheck)
438 if self.config.useConvergence:
439 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
440 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
443 if self.config.useConvergence:
444 self.log.info(
"Coadd patch %s subregion %s finished with " 445 "convergence metric %s after %s iterations",
446 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
448 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
449 skyInfo.patchInfo.getIndex(), subIter, modelIter)
450 if self.config.useConvergence
and convergenceMetric > 0:
451 self.log.info(
"Final convergence improvement was %.4f%% overall",
452 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
454 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, tempExpRefList, weightList,
455 calibration=self.scaleZeroPoint.getCalib(),
456 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
458 variance=baseVariance)
460 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
461 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
463 def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl):
464 """Calculate the number of exposures contributing to each subfilter. 468 dcrModels : `lsst.pipe.tasks.DcrModel` 469 Best fit model of the true sky after correcting chromatic effects. 470 bbox : `lsst.afw.geom.box.Box2I` 471 Bounding box of the patch to coadd. 472 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 473 The data references to the input warped exposures. 474 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 475 Each element of the `dict` contains the new mask plane name 476 (e.g. "CLIPPED and/or "NO_DATA") as the key, 477 and the list of SpanSets to apply to the mask. 478 statsCtrl : `lsst.afw.math.StatisticsControl` 479 Statistics control object for coadd 483 dcrNImages : `list` of `lsst.afw.image.ImageU` 484 List of exposure count images for each subfilter 485 dcrWeights : `list` of `lsst.afw.image.ImageF` 486 Per-pixel weights for each subfilter. 487 Equal to the number of unmasked images contributing to each pixel. 489 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
490 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
492 for tempExpRef, altMaskSpans
in zip(tempExpRefList, spanSetMaskList):
493 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
494 visitInfo = exposure.getInfo().getVisitInfo()
495 wcs = exposure.getInfo().getWcs()
497 if altMaskSpans
is not None:
499 weightImage = np.zeros_like(exposure.image.array)
500 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
501 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
502 for dcr, dcrNImage, dcrWeight
in zip(dcrShift, dcrNImages, dcrWeights):
504 shiftedWeights = applyDcr(weightImage, dcr, useInverse=
True,
505 order=self.config.imageInterpOrder)
506 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
507 dcrWeight.array += shiftedWeights
509 weightsThreshold = 1.
510 goodPix = dcrWeights[0].array > weightsThreshold
511 for weights
in dcrWeights[1:]:
512 goodPix = (weights.array > weightsThreshold) & goodPix
513 for subfilter
in range(self.config.dcrNumSubfilters):
514 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
515 dcrWeights[subfilter].array[~goodPix] = 0.
516 dcrNImages[subfilter].array[~goodPix] = 0
517 return (dcrNImages, dcrWeights)
520 statsCtrl, convergenceMetric,
521 baseMask, gain, modelWeights, refImage, dcrWeights):
522 """Assemble the DCR coadd for a sub-region. 524 Build a DCR-matched template for each input exposure, then shift the 525 residuals according to the DCR in each subfilter. 526 Stack the shifted residuals and apply them as a correction to the 527 solution from the previous iteration. 528 Restrict the new model solutions from varying by more than a factor of 529 `modelClampFactor` from the last solution, and additionally restrict the 530 individual subfilter models from varying by more than a factor of 531 `frequencyClampFactor` from their average. 532 Finally, mitigate potentially oscillating solutions by averaging the new 533 solution with the solution from the previous iteration, weighted by 534 their convergence metric. 538 dcrModels : `lsst.pipe.tasks.DcrModel` 539 Best fit model of the true sky after correcting chromatic effects. 540 subExposures : `dict` of `lsst.afw.image.ExposureF` 541 The pre-loaded exposures for the current subregion. 542 bbox : `lsst.afw.geom.box.Box2I` 543 Bounding box of the subregion to coadd. 544 dcrBBox : `lsst.afw.geom.box.Box2I` 545 Sub-region of the coadd which includes a buffer to allow for DCR. 546 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 547 The data references to the input warped exposures. 548 statsCtrl : `lsst.afw.math.StatisticsControl` 549 Statistics control object for coadd 550 convergenceMetric : `float` 551 Quality of fit metric for the matched templates of the input images. 552 baseMask : `lsst.afw.image.Mask` 553 Mask of the initial template coadd. 554 gain : `float`, optional 555 Relative weight to give the new solution when updating the model. 556 modelWeights : `numpy.ndarray` or `float` 557 A 2D array of weight values that tapers smoothly to zero away from detected sources. 558 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 559 refImage : `lsst.afw.image.Image` 560 A reference image used to supply the default pixel values. 561 dcrWeights : `list` of `lsst.afw.image.Image` 562 Per-pixel weights for each subfilter. 563 Equal to the number of unmasked images contributing to each pixel. 565 residualGeneratorList = []
567 for tempExpRef
in tempExpRefList:
568 exposure = subExposures[tempExpRef.dataId[
"visit"]]
569 visitInfo = exposure.getInfo().getVisitInfo()
570 wcs = exposure.getInfo().getWcs()
571 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
572 order=self.config.imageInterpOrder,
573 splitSubfilters=self.config.splitSubfilters)
574 residual = exposure.image.array - templateImage.array
576 residual *= exposure.variance.array
580 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
582 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
583 mask=baseMask, gain=gain,
584 modelWeights=modelWeights,
586 dcrWeights=dcrWeights)
587 dcrModels.assign(dcrSubModelOut, bbox)
590 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 594 residual : `numpy.ndarray` 595 The residual masked image for one exposure, 596 after subtracting the matched template 597 visitInfo : `lsst.afw.image.VisitInfo` 598 Metadata for the exposure. 599 wcs : `lsst.afw.geom.SkyWcs` 600 Coordinate system definition (wcs) for the exposure. 601 filterInfo : `lsst.afw.image.Filter` 602 The filter definition, set in the current instruments' obs package. 603 Required for any calculation of DCR, including making matched templates. 607 residualImage : `numpy.ndarray` 608 The residual image for the next subfilter, shifted for DCR. 610 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
612 yield applyDcr(residual, dcr, useInverse=
True, order=self.config.imageInterpOrder)
615 mask, gain, modelWeights, refImage, dcrWeights):
616 """Calculate a new DcrModel from a set of image residuals. 620 dcrModels : `lsst.pipe.tasks.DcrModel` 621 Current model of the true sky after correcting chromatic effects. 622 residualGeneratorList : `generator` of `numpy.ndarray` 623 The residual image for the next subfilter, shifted for DCR. 624 dcrBBox : `lsst.afw.geom.box.Box2I` 625 Sub-region of the coadd which includes a buffer to allow for DCR. 626 statsCtrl : `lsst.afw.math.StatisticsControl` 627 Statistics control object for coadd 628 mask : `lsst.afw.image.Mask` 629 Mask to use for each new model image. 631 Relative weight to give the new solution when updating the model. 632 modelWeights : `numpy.ndarray` or `float` 633 A 2D array of weight values that tapers smoothly to zero away from detected sources. 634 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 635 refImage : `lsst.afw.image.Image` 636 A reference image used to supply the default pixel values. 637 dcrWeights : `list` of `lsst.afw.image.Image` 638 Per-pixel weights for each subfilter. 639 Equal to the number of unmasked images contributing to each pixel. 643 dcrModel : `lsst.pipe.tasks.DcrModel` 644 New model of the true sky after correcting chromatic effects. 647 for subfilter, model
in enumerate(dcrModels):
648 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
649 residual = np.sum(residualsList, axis=0)
650 residual *= dcrWeights[subfilter][dcrBBox].array
652 newModel = model[dcrBBox].clone()
653 newModel.array += residual
655 badPixels = ~np.isfinite(newModel.array)
656 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
657 if self.config.regularizeModelIterations > 0:
658 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
659 self.config.regularizeModelIterations,
660 self.config.regularizationWidth)
661 newModelImages.append(newModel)
662 if self.config.regularizeModelFrequency > 0:
663 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
664 self.config.regularizeModelFrequency,
665 self.config.regularizationWidth,
667 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
669 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
670 dcrModels.mask, dcrModels.variance)
673 """Calculate a quality of fit metric for the matched templates. 677 dcrModels : `lsst.pipe.tasks.DcrModel` 678 Best fit model of the true sky after correcting chromatic effects. 679 subExposures : `dict` of `lsst.afw.image.ExposureF` 680 The pre-loaded exposures for the current subregion. 681 bbox : `lsst.afw.geom.box.Box2I` 683 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 684 The data references to the input warped exposures. 685 weightList : `list` of `float` 686 The weight to give each input exposure in the coadd 687 statsCtrl : `lsst.afw.math.StatisticsControl` 688 Statistics control object for coadd 692 convergenceMetric : `float` 693 Quality of fit metric for all input exposures, within the sub-region 695 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
697 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
699 if np.max(significanceImage) == 0:
700 significanceImage += 1.
704 for tempExpRef, expWeight
in zip(tempExpRefList, weightList):
705 exposure = subExposures[tempExpRef.dataId[
"visit"]][bbox]
707 metric += singleMetric
708 metricList[tempExpRef.dataId[
"visit"]] = singleMetric
710 self.log.info(
"Individual metrics:\n%s", metricList)
711 return 1.0
if weight == 0.0
else metric/weight
714 """Calculate a quality of fit metric for a single matched template. 718 dcrModels : `lsst.pipe.tasks.DcrModel` 719 Best fit model of the true sky after correcting chromatic effects. 720 exposure : `lsst.afw.image.ExposureF` 721 The input warped exposure to evaluate. 722 significanceImage : `numpy.ndarray` 723 Array of weights for each pixel corresponding to its significance 724 for the convergence calculation. 725 statsCtrl : `lsst.afw.math.StatisticsControl` 726 Statistics control object for coadd 730 convergenceMetric : `float` 731 Quality of fit metric for one exposure, within the sub-region. 733 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
734 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure, order=self.config.imageInterpOrder)
735 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
736 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
738 finitePixels = np.isfinite(diffVals)
739 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
740 convergeMaskPixels = exposure.mask.array & convergeMask > 0
741 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
742 if np.sum(usePixels) == 0:
745 diffUse = diffVals[usePixels]
746 refUse = refVals[usePixels]
747 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
751 """Add a list of sub-band coadds together. 755 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 756 A list of coadd exposures, each exposure containing 757 the model for one subfilter. 761 coaddExposure : `lsst.afw.image.ExposureF` 762 A single coadd exposure that is the sum of the sub-bands. 764 coaddExposure = dcrCoadds[0].clone()
765 for coadd
in dcrCoadds[1:]:
766 coaddExposure.maskedImage += coadd.maskedImage
769 def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None,
770 mask=None, variance=None):
771 """Create a list of coadd exposures from a list of masked images. 775 dcrModels : `lsst.pipe.tasks.DcrModel` 776 Best fit model of the true sky after correcting chromatic effects. 777 skyInfo : `lsst.pipe.base.Struct` 778 Patch geometry information, from getSkyInfo 779 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 780 The data references to the input warped exposures. 781 weightList : `list` of `float` 782 The weight to give each input exposure in the coadd 783 calibration : `lsst.afw.Image.Calib`, optional 784 Scale factor to set the photometric zero point of an exposure. 785 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 786 A record of the observations that are included in the coadd. 787 mask : `lsst.afw.image.Mask`, optional 788 Optional mask to override the values in the final coadd. 789 variance : `lsst.afw.image.Image`, optional 790 Optional variance plane to override the values in the final coadd. 794 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 795 A list of coadd exposures, each exposure containing 796 the model for one subfilter. 799 for model
in dcrModels:
800 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
801 if calibration
is not None:
802 coaddExposure.setCalib(calibration)
803 if coaddInputs
is not None:
804 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
807 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
808 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
809 maskedImage.image = model
810 maskedImage.mask = dcrModels.mask
811 maskedImage.variance = dcrModels.variance
812 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
814 coaddExposure.setMask(mask)
815 if variance
is not None:
816 coaddExposure.setVariance(variance)
817 dcrCoadds.append(coaddExposure)
821 """Calculate the gain to use for the current iteration. 823 After calculating a new DcrModel, each value is averaged with the 824 value in the corresponding pixel from the previous iteration. This 825 reduces oscillating solutions that iterative techniques are plagued by, 826 and speeds convergence. By far the biggest changes to the model 827 happen in the first couple iterations, so we can also use a more 828 aggressive gain later when the model is changing slowly. 832 convergenceList : `list` of `float` 833 The quality of fit metric from each previous iteration. 834 gainList : `list` of `float` 835 The gains used in each previous iteration: appended with the new 837 Gains are numbers between ``self.config.baseGain`` and 1. 842 Relative weight to give the new solution when updating the model. 843 A value of 1.0 gives equal weight to both solutions. 848 If ``len(convergenceList) != len(gainList)+1``. 850 nIter = len(convergenceList)
851 if nIter != len(gainList) + 1:
852 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 853 % (len(convergenceList), len(gainList)))
855 if self.config.baseGain
is None:
858 baseGain = 1./(self.config.dcrNumSubfilters - 1)
860 baseGain = self.config.baseGain
862 if self.config.useProgressiveGain
and nIter > 2:
870 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
871 for i
in range(nIter - 1)]
874 estFinalConv = np.array(estFinalConv)
875 estFinalConv[estFinalConv < 0] = 0
877 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
878 lastGain = gainList[-1]
879 lastConv = convergenceList[-2]
880 newConv = convergenceList[-1]
885 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
891 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
892 newGain = 1 - abs(delta)
894 newGain = (newGain + lastGain)/2.
895 gain = max(baseGain, newGain)
898 gainList.append(gain)
902 """Build an array that smoothly tapers to 0 away from detected sources. 906 dcrModels : `lsst.pipe.tasks.DcrModel` 907 Best fit model of the true sky after correcting chromatic effects. 908 dcrBBox : `lsst.afw.geom.box.Box2I` 909 Sub-region of the coadd which includes a buffer to allow for DCR. 913 weights : `numpy.ndarray` or `float` 914 A 2D array of weight values that tapers smoothly to zero away from detected sources. 915 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 920 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 922 if not self.config.useModelWeights:
924 if self.config.modelWeightsWidth < 0:
925 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
926 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
927 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
928 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
929 weights[convergeMaskPixels] = 1.
930 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
931 weights /= np.max(weights)
935 """Smoothly replace model pixel values with those from a 936 reference at locations away from detected sources. 940 modelImages : `list` of `lsst.afw.image.Image` 941 The new DCR model images from the current iteration. 942 The values will be modified in place. 943 refImage : `lsst.afw.image.MaskedImage` 944 A reference image used to supply the default pixel values. 945 modelWeights : `numpy.ndarray` or `float` 946 A 2D array of weight values that tapers smoothly to zero away from detected sources. 947 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 949 if self.config.useModelWeights:
950 for model
in modelImages:
951 model.array *= modelWeights
952 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
954 def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList):
955 """Pre-load sub-regions of a list of exposures. 959 bbox : `lsst.afw.geom.box.Box2I` 961 statsCtrl : `lsst.afw.math.StatisticsControl` 962 Statistics control object for coadd 963 tempExpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 964 The data references to the input warped exposures. 965 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 966 The image scalars correct for the zero point of the exposures. 967 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 968 Each element is dict with keys = mask plane name to add the spans to 972 subExposures : `dict` 973 The `dict` keys are the visit IDs, 974 and the values are `lsst.afw.image.ExposureF` 975 The pre-loaded exposures for the current subregion. 976 The variance plane contains weights, and not the variance 979 zipIterables = zip(tempExpRefList, imageScalerList, spanSetMaskList)
981 for tempExpRef, imageScaler, altMaskSpans
in zipIterables:
982 exposure = tempExpRef.get(tempExpName +
"_sub", bbox=bbox)
983 if altMaskSpans
is not None:
985 imageScaler.scaleMaskedImage(exposure.maskedImage)
987 exposure.variance.array[:, :] = 0.
989 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
992 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
993 subExposures[tempExpRef.dataId[
"visit"]] = exposure
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, tempExpRefList, spanSetMaskList, statsCtrl)
def calculateConvergence(self, dcrModels, subExposures, bbox, tempExpRefList, weightList, statsCtrl)
def fillCoadd(self, dcrModels, skyInfo, tempExpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, tempExpRefList, statsCtrl, convergenceMetric, baseMask, gain, modelWeights, refImage, dcrWeights)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def run(self, skyInfo, tempExpRefList, imageScalerList, weightList, supplementaryData=None)
def prepareDcrInputs(self, templateCoadd, tempExpRefList, weightList)
def processResults(self, coaddExposure, dataRef)
def calculateGain(self, convergenceList, gainList)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, mask, gain, modelWeights, refImage, dcrWeights)
def _subBBoxIter(bbox, subregionSize)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)
def loadSubExposures(self, bbox, statsCtrl, tempExpRefList, imageScalerList, spanSetMaskList)