25 from scipy
import ndimage
35 from .assembleCoadd
import AssembleCoaddTask, CompareWarpAssembleCoaddTask, CompareWarpAssembleCoaddConfig
36 from .measurePsf
import MeasurePsfTask
38 __all__ = [
"DcrAssembleCoaddTask",
"DcrAssembleCoaddConfig"]
42 dcrNumSubfilters = pexConfig.Field(
44 doc=
"Number of sub-filters to forward model chromatic effects to fit the supplied exposures.",
47 maxNumIter = pexConfig.Field(
50 doc=
"Maximum number of iterations of forward modeling.",
53 minNumIter = pexConfig.Field(
56 doc=
"Minimum number of iterations of forward modeling.",
59 convergenceThreshold = pexConfig.Field(
61 doc=
"Target relative change in convergence between iterations of forward modeling.",
64 useConvergence = pexConfig.Field(
66 doc=
"Use convergence test as a forward modeling end condition?" 67 "If not set, skips calculating convergence and runs for ``maxNumIter`` iterations",
70 baseGain = pexConfig.Field(
73 doc=
"Relative weight to give the new solution vs. the last solution when updating the model." 74 "A value of 1.0 gives equal weight to both solutions." 75 "Small values imply slower convergence of the solution, but can " 76 "help prevent overshooting and failures in the fit." 77 "If ``baseGain`` is None, a conservative gain " 78 "will be calculated from the number of subfilters. ",
81 useProgressiveGain = pexConfig.Field(
83 doc=
"Use a gain that slowly increases above ``baseGain`` to accelerate convergence? " 84 "When calculating the next gain, we use up to 5 previous gains and convergence values." 85 "Can be set to False to force the model to change at the rate of ``baseGain``. ",
88 doAirmassWeight = pexConfig.Field(
90 doc=
"Weight exposures by airmass? Useful if there are relatively few high-airmass observations.",
93 modelWeightsWidth = pexConfig.Field(
95 doc=
"Width of the region around detected sources to include in the DcrModel.",
98 useModelWeights = pexConfig.Field(
100 doc=
"Width of the region around detected sources to include in the DcrModel.",
103 splitSubfilters = pexConfig.Field(
105 doc=
"Calculate DCR for two evenly-spaced wavelengths in each subfilter." 106 "Instead of at the midpoint",
109 regularizeModelIterations = pexConfig.Field(
111 doc=
"Maximum relative change of the model allowed between iterations." 112 "Set to zero to disable.",
115 regularizeModelFrequency = pexConfig.Field(
117 doc=
"Maximum relative change of the model allowed between subfilters." 118 "Set to zero to disable.",
121 convergenceMaskPlanes = pexConfig.ListField(
123 default=[
"DETECTED"],
124 doc=
"Mask planes to use to calculate convergence." 126 regularizationWidth = pexConfig.Field(
129 doc=
"Minimum radius of a region to include in regularization, in pixels." 131 imageInterpOrder = pexConfig.Field(
133 doc=
"The order of the spline interpolation used to shift the image plane.",
136 accelerateModel = pexConfig.Field(
138 doc=
"Factor to amplify the differences between model planes by to speed convergence.",
141 doCalculatePsf = pexConfig.Field(
143 doc=
"Set to detect stars and recalculate the PSF from the final coadd." 144 "Otherwise the PSF is estimated from a selection of the best input exposures",
147 detectPsfSources = pexConfig.ConfigurableField(
148 target=measAlg.SourceDetectionTask,
149 doc=
"Task to detect sources for PSF measurement, if ``doCalculatePsf`` is set.",
151 measurePsfSources = pexConfig.ConfigurableField(
152 target=SingleFrameMeasurementTask,
153 doc=
"Task to measure sources for PSF measurement, if ``doCalculatePsf`` is set." 155 measurePsf = pexConfig.ConfigurableField(
156 target=MeasurePsfTask,
157 doc=
"Task to measure the PSF of the coadd, if ``doCalculatePsf`` is set.",
161 CompareWarpAssembleCoaddConfig.setDefaults(self)
180 self.
measurePsf.starSelector[
"objectSize"].doFluxLimit =
False 184 """Assemble DCR coadded images from a set of warps. 189 The number of pixels to grow each subregion by to allow for DCR. 193 As with AssembleCoaddTask, we want to assemble a coadded image from a set of 194 Warps (also called coadded temporary exposures), including the effects of 195 Differential Chromatic Refraction (DCR). 196 For full details of the mathematics and algorithm, please see 197 DMTN-037: DCR-matched template generation (https://dmtn-037.lsst.io). 199 This Task produces a DCR-corrected deepCoadd, as well as a dcrCoadd for 200 each subfilter used in the iterative calculation. 201 It begins by dividing the bandpass-defining filter into N equal bandwidth 202 "subfilters", and divides the flux in each pixel from an initial coadd 203 equally into each as a "dcrModel". Because the airmass and parallactic 204 angle of each individual exposure is known, we can calculate the shift 205 relative to the center of the band in each subfilter due to DCR. For each 206 exposure we apply this shift as a linear transformation to the dcrModels 207 and stack the results to produce a DCR-matched exposure. The matched 208 exposures are subtracted from the input exposures to produce a set of 209 residual images, and these residuals are reverse shifted for each 210 exposures' subfilters and stacked. The shifted and stacked residuals are 211 added to the dcrModels to produce a new estimate of the flux in each pixel 212 within each subfilter. The dcrModels are solved for iteratively, which 213 continues until the solution from a new iteration improves by less than 214 a set percentage, or a maximum number of iterations is reached. 215 Two forms of regularization are employed to reduce unphysical results. 216 First, the new solution is averaged with the solution from the previous 217 iteration, which mitigates oscillating solutions where the model 218 overshoots with alternating very high and low values. 219 Second, a common degeneracy when the data have a limited range of airmass or 220 parallactic angle values is for one subfilter to be fit with very low or 221 negative values, while another subfilter is fit with very high values. This 222 typically appears in the form of holes next to sources in one subfilter, 223 and corresponding extended wings in another. Because each subfilter has 224 a narrow bandwidth we assume that physical sources that are above the noise 225 level will not vary in flux by more than a factor of `frequencyClampFactor` 226 between subfilters, and pixels that have flux deviations larger than that 227 factor will have the excess flux distributed evenly among all subfilters. 230 ConfigClass = DcrAssembleCoaddConfig
231 _DefaultName =
"dcrAssembleCoadd" 235 if self.config.doCalculatePsf:
236 self.
schema = afwTable.SourceTable.makeMinimalSchema()
237 self.makeSubtask(
"detectPsfSources", schema=self.
schema)
238 self.makeSubtask(
"measurePsfSources", schema=self.
schema)
239 self.makeSubtask(
"measurePsf", schema=self.
schema)
242 def runDataRef(self, dataRef, selectDataList=None, warpRefList=None):
243 """Assemble a coadd from a set of warps. 245 Coadd a set of Warps. Compute weights to be applied to each Warp and 246 find scalings to match the photometric zeropoint to a reference Warp. 247 Assemble the Warps using run method. 248 Forward model chromatic effects across multiple subfilters, 249 and subtract from the input Warps to build sets of residuals. 250 Use the residuals to construct a new ``DcrModel`` for each subfilter, 251 and iterate until the model converges. 252 Interpolate over NaNs and optionally write the coadd to disk. 253 Return the coadded exposure. 257 dataRef : `lsst.daf.persistence.ButlerDataRef` 258 Data reference defining the patch for coaddition and the 260 selectDataList : `list` of `lsst.daf.persistence.ButlerDataRef` 261 List of data references to warps. Data to be coadded will be 262 selected from this list based on overlap with the patch defined by 267 results : `lsst.pipe.base.Struct` 268 The Struct contains the following fields: 270 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 271 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 272 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 273 - ``dcrNImages``: `list` of exposure count images for each subfilter 275 if (selectDataList
is None and warpRefList
is None)
or (selectDataList
and warpRefList):
276 raise RuntimeError(
"runDataRef must be supplied either a selectDataList or warpRefList")
278 results = AssembleCoaddTask.runDataRef(self, dataRef, selectDataList=selectDataList,
279 warpRefList=warpRefList)
282 self.log.warn(
"Could not construct DcrModel for patch %s: no data to coadd.",
283 skyInfo.patchInfo.getIndex())
285 for subfilter
in range(self.config.dcrNumSubfilters):
287 results.dcrCoadds[subfilter].setPsf(results.coaddExposure.getPsf())
288 AssembleCoaddTask.processResults(self, results.dcrCoadds[subfilter], dataRef)
289 if self.config.doWrite:
290 self.log.info(
"Persisting dcrCoadd")
291 dataRef.put(results.dcrCoadds[subfilter],
"dcrCoadd", subfilter=subfilter,
292 numSubfilters=self.config.dcrNumSubfilters)
293 if self.config.doNImage
and results.dcrNImages
is not None:
294 dataRef.put(results.dcrNImages[subfilter],
"dcrCoadd_nImage", subfilter=subfilter,
295 numSubfilters=self.config.dcrNumSubfilters)
300 """Interpolate over missing data and mask bright stars. 302 Also detect sources on the coadd exposure and measure the final PSF, 303 if ``doCalculatePsf`` is set. 307 coaddExposure : `lsst.afw.image.Exposure` 308 The final coadded exposure. 309 dataRef : `lsst.daf.persistence.ButlerDataRef` 310 Data reference defining the patch for coaddition and the 315 if self.config.doCalculatePsf:
316 expId = dataRef.get(
"dcrCoaddId")
317 table = afwTable.SourceTable.make(self.
schema)
318 detResults = self.detectPsfSources.
run(table, coaddExposure, expId, clearMask=
False)
319 coaddSources = detResults.sources
320 self.measurePsfSources.
run(
321 measCat=coaddSources,
322 exposure=coaddExposure,
326 psfResults = self.measurePsf.
run(coaddExposure, coaddSources, expId=expId)
327 except RuntimeError
as e:
328 self.log.warn(
"Unable to calculate PSF, using default coadd PSF: %s" % e)
330 coaddExposure.setPsf(psfResults.psf)
333 """Prepare the DCR coadd by iterating through the visitInfo of the input warps. 335 Sets the property ``bufferSize``. 339 templateCoadd : `lsst.afw.image.ExposureF` 340 The initial coadd exposure before accounting for DCR. 341 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 342 The data references to the input warped exposures. 343 weightList : `list` of `float` 344 The weight to give each input exposure in the coadd 345 Will be modified in place if ``doAirmassWeight`` is set. 349 dcrModels : `lsst.pipe.tasks.DcrModel` 350 Best fit model of the true sky after correcting chromatic effects. 355 If ``lambdaMin`` is missing from the Mapper class of the obs package being used. 357 filterInfo = templateCoadd.getFilter()
358 if np.isnan(filterInfo.getFilterProperty().getLambdaMin()):
359 raise NotImplementedError(
"No minimum/maximum wavelength information found" 360 " in the filter definition! Please add lambdaMin and lambdaMax" 361 " to the Mapper class in your obs package.")
366 for visitNum, warpExpRef
in enumerate(warpRefList):
367 visitInfo = warpExpRef.get(tempExpName +
"_visitInfo")
368 visit = warpExpRef.dataId[
"visit"]
369 airmass = visitInfo.getBoresightAirmass()
370 parallacticAngle = visitInfo.getBoresightParAngle().asDegrees()
371 airmassDict[visit] = airmass
372 angleDict[visit] = parallacticAngle
373 if self.config.doAirmassWeight:
374 weightList[visitNum] *= airmass
375 dcrShifts.append(np.max(np.abs(calculateDcr(visitInfo, templateCoadd.getWcs(),
376 filterInfo, self.config.dcrNumSubfilters))))
377 self.log.info(
"Selected airmasses:\n%s", airmassDict)
378 self.log.info(
"Selected parallactic angles:\n%s", angleDict)
381 dcrModels = DcrModel.fromImage(templateCoadd.maskedImage,
382 self.config.dcrNumSubfilters,
383 filterInfo=filterInfo,
387 def run(self, skyInfo, warpRefList, imageScalerList, weightList,
388 supplementaryData=None):
389 """Assemble the coadd. 391 Requires additional inputs Struct ``supplementaryData`` to contain a 392 ``templateCoadd`` that serves as the model of the static sky. 394 Find artifacts and apply them to the warps' masks creating a list of 395 alternative masks with a new "CLIPPED" plane and updated "NO_DATA" plane 396 Then pass these alternative masks to the base class's assemble method. 398 Divide the ``templateCoadd`` evenly between each subfilter of a 399 ``DcrModel`` as the starting best estimate of the true wavelength- 400 dependent sky. Forward model the ``DcrModel`` using the known 401 chromatic effects in each subfilter and calculate a convergence metric 402 based on how well the modeled template matches the input warps. If 403 the convergence has not yet reached the desired threshold, then shift 404 and stack the residual images to build a new ``DcrModel``. Apply 405 conditioning to prevent oscillating solutions between iterations or 408 Once the ``DcrModel`` reaches convergence or the maximum number of 409 iterations has been reached, fill the metadata for each subfilter 410 image and make them proper ``coaddExposure``s. 414 skyInfo : `lsst.pipe.base.Struct` 415 Patch geometry information, from getSkyInfo 416 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 417 The data references to the input warped exposures. 418 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 419 The image scalars correct for the zero point of the exposures. 420 weightList : `list` of `float` 421 The weight to give each input exposure in the coadd 422 supplementaryData : `lsst.pipe.base.Struct` 423 Result struct returned by ``makeSupplementaryData`` with components: 425 - ``templateCoadd``: coadded exposure (`lsst.afw.image.Exposure`) 429 result : `lsst.pipe.base.Struct` 430 Result struct with components: 432 - ``coaddExposure``: coadded exposure (`lsst.afw.image.Exposure`) 433 - ``nImage``: exposure count image (`lsst.afw.image.ImageU`) 434 - ``dcrCoadds``: `list` of coadded exposures for each subfilter 435 - ``dcrNImages``: `list` of exposure count images for each subfilter 437 minNumIter = self.config.minNumIter
or self.config.dcrNumSubfilters
438 maxNumIter = self.config.maxNumIter
or self.config.dcrNumSubfilters*3
439 templateCoadd = supplementaryData.templateCoadd
440 baseMask = templateCoadd.mask.clone()
443 baseVariance = templateCoadd.variance.clone()
444 baseVariance /= self.config.dcrNumSubfilters
445 spanSetMaskList = self.
findArtifacts(templateCoadd, warpRefList, imageScalerList)
447 templateCoadd.setMask(baseMask)
448 badMaskPlanes = self.config.badMaskPlanes[:]
453 badPixelMask = templateCoadd.mask.getPlaneBitMask(badMaskPlanes)
457 if self.config.doNImage:
458 dcrNImages, dcrWeights = self.
calculateNImage(dcrModels, skyInfo.bbox, warpRefList,
459 spanSetMaskList, stats.ctrl)
460 nImage = afwImage.ImageU(skyInfo.bbox)
464 for dcrNImage
in dcrNImages:
469 subregionSize = afwGeom.Extent2I(*self.config.subregionSize)
470 nSubregions = (ceil(skyInfo.bbox.getHeight()/subregionSize[1]) *
471 ceil(skyInfo.bbox.getWidth()/subregionSize[0]))
473 for subBBox
in self.
_subBBoxIter(skyInfo.bbox, subregionSize):
476 self.log.info(
"Computing coadd over patch %s subregion %s of %s: %s",
477 skyInfo.patchInfo.getIndex(), subIter, nSubregions, subBBox)
478 dcrBBox = afwGeom.Box2I(subBBox)
480 dcrBBox.clip(dcrModels.bbox)
483 imageScalerList, spanSetMaskList)
485 warpRefList, weightList, stats.ctrl)
486 self.log.info(
"Initial convergence : %s", convergenceMetric)
487 convergenceList = [convergenceMetric]
489 convergenceCheck = 1.
490 refImage = templateCoadd.image
491 while (convergenceCheck > self.config.convergenceThreshold
or modelIter <= minNumIter):
494 stats.ctrl, convergenceMetric, gain,
495 modelWeights, refImage, dcrWeights)
496 if self.config.useConvergence:
498 warpRefList, weightList, stats.ctrl)
499 if convergenceMetric == 0:
500 self.log.warn(
"Coadd patch %s subregion %s had convergence metric of 0.0 which is " 501 "most likely due to there being no valid data in the region.",
502 skyInfo.patchInfo.getIndex(), subIter)
504 convergenceCheck = (convergenceList[-1] - convergenceMetric)/convergenceMetric
505 if (convergenceCheck < 0) & (modelIter > minNumIter):
506 self.log.warn(
"Coadd patch %s subregion %s diverged before reaching maximum " 507 "iterations or desired convergence improvement of %s." 509 skyInfo.patchInfo.getIndex(), subIter,
510 self.config.convergenceThreshold, convergenceCheck)
512 convergenceList.append(convergenceMetric)
513 if modelIter > maxNumIter:
514 if self.config.useConvergence:
515 self.log.warn(
"Coadd patch %s subregion %s reached maximum iterations " 516 "before reaching desired convergence improvement of %s." 517 " Final convergence improvement: %s",
518 skyInfo.patchInfo.getIndex(), subIter,
519 self.config.convergenceThreshold, convergenceCheck)
522 if self.config.useConvergence:
523 self.log.info(
"Iteration %s with convergence metric %s, %.4f%% improvement (gain: %.2f)",
524 modelIter, convergenceMetric, 100.*convergenceCheck, gain)
527 if self.config.useConvergence:
528 self.log.info(
"Coadd patch %s subregion %s finished with " 529 "convergence metric %s after %s iterations",
530 skyInfo.patchInfo.getIndex(), subIter, convergenceMetric, modelIter)
532 self.log.info(
"Coadd patch %s subregion %s finished after %s iterations",
533 skyInfo.patchInfo.getIndex(), subIter, modelIter)
534 if self.config.useConvergence
and convergenceMetric > 0:
535 self.log.info(
"Final convergence improvement was %.4f%% overall",
536 100*(convergenceList[0] - convergenceMetric)/convergenceMetric)
538 dcrCoadds = self.
fillCoadd(dcrModels, skyInfo, warpRefList, weightList,
539 calibration=self.scaleZeroPoint.getPhotoCalib(),
540 coaddInputs=templateCoadd.getInfo().getCoaddInputs(),
542 variance=baseVariance)
544 return pipeBase.Struct(coaddExposure=coaddExposure, nImage=nImage,
545 dcrCoadds=dcrCoadds, dcrNImages=dcrNImages)
548 """Calculate the number of exposures contributing to each subfilter. 552 dcrModels : `lsst.pipe.tasks.DcrModel` 553 Best fit model of the true sky after correcting chromatic effects. 554 bbox : `lsst.afw.geom.box.Box2I` 555 Bounding box of the patch to coadd. 556 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 557 The data references to the input warped exposures. 558 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 559 Each element of the `dict` contains the new mask plane name 560 (e.g. "CLIPPED and/or "NO_DATA") as the key, 561 and the list of SpanSets to apply to the mask. 562 statsCtrl : `lsst.afw.math.StatisticsControl` 563 Statistics control object for coadd 567 dcrNImages : `list` of `lsst.afw.image.ImageU` 568 List of exposure count images for each subfilter 569 dcrWeights : `list` of `lsst.afw.image.ImageF` 570 Per-pixel weights for each subfilter. 571 Equal to 1/(number of unmasked images contributing to each pixel). 573 dcrNImages = [afwImage.ImageU(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
574 dcrWeights = [afwImage.ImageF(bbox)
for subfilter
in range(self.config.dcrNumSubfilters)]
576 for warpExpRef, altMaskSpans
in zip(warpRefList, spanSetMaskList):
577 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
578 visitInfo = exposure.getInfo().getVisitInfo()
579 wcs = exposure.getInfo().getWcs()
581 if altMaskSpans
is not None:
583 weightImage = np.zeros_like(exposure.image.array)
584 weightImage[(mask.array & statsCtrl.getAndMask()) == 0] = 1.
585 dcrShift = calculateDcr(visitInfo, wcs, dcrModels.filter, self.config.dcrNumSubfilters)
586 for dcr, dcrNImage, dcrWeight
in zip(dcrShift, dcrNImages, dcrWeights):
588 shiftedWeights = applyDcr(weightImage, dcr, useInverse=
True,
589 order=self.config.imageInterpOrder)
590 dcrNImage.array += np.rint(shiftedWeights).astype(dcrNImage.array.dtype)
591 dcrWeight.array += shiftedWeights
593 weightsThreshold = 1.
594 goodPix = dcrWeights[0].array > weightsThreshold
595 for weights
in dcrWeights[1:]:
596 goodPix = (weights.array > weightsThreshold) & goodPix
597 for subfilter
in range(self.config.dcrNumSubfilters):
598 dcrWeights[subfilter].array[goodPix] = 1./dcrWeights[subfilter].array[goodPix]
599 dcrWeights[subfilter].array[~goodPix] = 0.
600 dcrNImages[subfilter].array[~goodPix] = 0
601 return (dcrNImages, dcrWeights)
604 statsCtrl, convergenceMetric,
605 gain, modelWeights, refImage, dcrWeights):
606 """Assemble the DCR coadd for a sub-region. 608 Build a DCR-matched template for each input exposure, then shift the 609 residuals according to the DCR in each subfilter. 610 Stack the shifted residuals and apply them as a correction to the 611 solution from the previous iteration. 612 Restrict the new model solutions from varying by more than a factor of 613 `modelClampFactor` from the last solution, and additionally restrict the 614 individual subfilter models from varying by more than a factor of 615 `frequencyClampFactor` from their average. 616 Finally, mitigate potentially oscillating solutions by averaging the new 617 solution with the solution from the previous iteration, weighted by 618 their convergence metric. 622 dcrModels : `lsst.pipe.tasks.DcrModel` 623 Best fit model of the true sky after correcting chromatic effects. 624 subExposures : `dict` of `lsst.afw.image.ExposureF` 625 The pre-loaded exposures for the current subregion. 626 bbox : `lsst.afw.geom.box.Box2I` 627 Bounding box of the subregion to coadd. 628 dcrBBox : `lsst.afw.geom.box.Box2I` 629 Sub-region of the coadd which includes a buffer to allow for DCR. 630 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 631 The data references to the input warped exposures. 632 statsCtrl : `lsst.afw.math.StatisticsControl` 633 Statistics control object for coadd 634 convergenceMetric : `float` 635 Quality of fit metric for the matched templates of the input images. 636 gain : `float`, optional 637 Relative weight to give the new solution when updating the model. 638 modelWeights : `numpy.ndarray` or `float` 639 A 2D array of weight values that tapers smoothly to zero away from detected sources. 640 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 641 refImage : `lsst.afw.image.Image` 642 A reference image used to supply the default pixel values. 643 dcrWeights : `list` of `lsst.afw.image.Image` 644 Per-pixel weights for each subfilter. 645 Equal to 1/(number of unmasked images contributing to each pixel). 647 residualGeneratorList = []
649 for warpExpRef
in warpRefList:
650 exposure = subExposures[warpExpRef.dataId[
"visit"]]
651 visitInfo = exposure.getInfo().getVisitInfo()
652 wcs = exposure.getInfo().getWcs()
653 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
654 order=self.config.imageInterpOrder,
655 splitSubfilters=self.config.splitSubfilters,
656 amplifyModel=self.config.accelerateModel)
657 residual = exposure.image.array - templateImage.array
659 residual *= exposure.variance.array
663 residualGeneratorList.append(self.
dcrResiduals(residual, visitInfo, wcs, dcrModels.filter))
665 dcrSubModelOut = self.
newModelFromResidual(dcrModels, residualGeneratorList, dcrBBox, statsCtrl,
667 modelWeights=modelWeights,
669 dcrWeights=dcrWeights)
670 dcrModels.assign(dcrSubModelOut, bbox)
673 """Prepare a residual image for stacking in each subfilter by applying the reverse DCR shifts. 677 residual : `numpy.ndarray` 678 The residual masked image for one exposure, 679 after subtracting the matched template 680 visitInfo : `lsst.afw.image.VisitInfo` 681 Metadata for the exposure. 682 wcs : `lsst.afw.geom.SkyWcs` 683 Coordinate system definition (wcs) for the exposure. 684 filterInfo : `lsst.afw.image.Filter` 685 The filter definition, set in the current instruments' obs package. 686 Required for any calculation of DCR, including making matched templates. 690 residualImage : `numpy.ndarray` 691 The residual image for the next subfilter, shifted for DCR. 693 dcrShift = calculateDcr(visitInfo, wcs, filterInfo, self.config.dcrNumSubfilters)
695 yield applyDcr(residual, dcr, useInverse=
True, order=self.config.imageInterpOrder)
698 gain, modelWeights, refImage, dcrWeights):
699 """Calculate a new DcrModel from a set of image residuals. 703 dcrModels : `lsst.pipe.tasks.DcrModel` 704 Current model of the true sky after correcting chromatic effects. 705 residualGeneratorList : `generator` of `numpy.ndarray` 706 The residual image for the next subfilter, shifted for DCR. 707 dcrBBox : `lsst.afw.geom.box.Box2I` 708 Sub-region of the coadd which includes a buffer to allow for DCR. 709 statsCtrl : `lsst.afw.math.StatisticsControl` 710 Statistics control object for coadd 712 Relative weight to give the new solution when updating the model. 713 modelWeights : `numpy.ndarray` or `float` 714 A 2D array of weight values that tapers smoothly to zero away from detected sources. 715 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 716 refImage : `lsst.afw.image.Image` 717 A reference image used to supply the default pixel values. 718 dcrWeights : `list` of `lsst.afw.image.Image` 719 Per-pixel weights for each subfilter. 720 Equal to 1/(number of unmasked images contributing to each pixel). 724 dcrModel : `lsst.pipe.tasks.DcrModel` 725 New model of the true sky after correcting chromatic effects. 728 for subfilter, model
in enumerate(dcrModels):
729 residualsList = [next(residualGenerator)
for residualGenerator
in residualGeneratorList]
730 residual = np.sum(residualsList, axis=0)
731 residual *= dcrWeights[subfilter][dcrBBox].array
733 newModel = model[dcrBBox].clone()
734 newModel.array += residual
736 badPixels = ~np.isfinite(newModel.array)
737 newModel.array[badPixels] = model[dcrBBox].array[badPixels]
738 if self.config.regularizeModelIterations > 0:
739 dcrModels.regularizeModelIter(subfilter, newModel, dcrBBox,
740 self.config.regularizeModelIterations,
741 self.config.regularizationWidth)
742 newModelImages.append(newModel)
743 if self.config.regularizeModelFrequency > 0:
744 dcrModels.regularizeModelFreq(newModelImages, dcrBBox, statsCtrl,
745 self.config.regularizeModelFrequency,
746 self.config.regularizationWidth)
747 dcrModels.conditionDcrModel(newModelImages, dcrBBox, gain=gain)
749 return DcrModel(newModelImages, dcrModels.filter, dcrModels.psf,
750 dcrModels.mask, dcrModels.variance)
753 """Calculate a quality of fit metric for the matched templates. 757 dcrModels : `lsst.pipe.tasks.DcrModel` 758 Best fit model of the true sky after correcting chromatic effects. 759 subExposures : `dict` of `lsst.afw.image.ExposureF` 760 The pre-loaded exposures for the current subregion. 761 bbox : `lsst.afw.geom.box.Box2I` 763 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 764 The data references to the input warped exposures. 765 weightList : `list` of `float` 766 The weight to give each input exposure in the coadd 767 statsCtrl : `lsst.afw.math.StatisticsControl` 768 Statistics control object for coadd 772 convergenceMetric : `float` 773 Quality of fit metric for all input exposures, within the sub-region 775 significanceImage = np.abs(dcrModels.getReferenceImage(bbox))
777 significanceImage += nSigma*dcrModels.calculateNoiseCutoff(dcrModels[1], statsCtrl,
779 if np.max(significanceImage) == 0:
780 significanceImage += 1.
784 for warpExpRef, expWeight
in zip(warpRefList, weightList):
785 exposure = subExposures[warpExpRef.dataId[
"visit"]][bbox]
787 metric += singleMetric
788 metricList[warpExpRef.dataId[
"visit"]] = singleMetric
790 self.log.info(
"Individual metrics:\n%s", metricList)
791 return 1.0
if weight == 0.0
else metric/weight
794 """Calculate a quality of fit metric for a single matched template. 798 dcrModels : `lsst.pipe.tasks.DcrModel` 799 Best fit model of the true sky after correcting chromatic effects. 800 exposure : `lsst.afw.image.ExposureF` 801 The input warped exposure to evaluate. 802 significanceImage : `numpy.ndarray` 803 Array of weights for each pixel corresponding to its significance 804 for the convergence calculation. 805 statsCtrl : `lsst.afw.math.StatisticsControl` 806 Statistics control object for coadd 810 convergenceMetric : `float` 811 Quality of fit metric for one exposure, within the sub-region. 813 convergeMask = exposure.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
814 templateImage = dcrModels.buildMatchedTemplate(exposure=exposure,
815 order=self.config.imageInterpOrder,
816 splitSubfilters=self.config.splitSubfilters,
817 amplifyModel=self.config.accelerateModel)
818 diffVals = np.abs(exposure.image.array - templateImage.array)*significanceImage
819 refVals = np.abs(exposure.image.array + templateImage.array)*significanceImage/2.
821 finitePixels = np.isfinite(diffVals)
822 goodMaskPixels = (exposure.mask.array & statsCtrl.getAndMask()) == 0
823 convergeMaskPixels = exposure.mask.array & convergeMask > 0
824 usePixels = finitePixels & goodMaskPixels & convergeMaskPixels
825 if np.sum(usePixels) == 0:
828 diffUse = diffVals[usePixels]
829 refUse = refVals[usePixels]
830 metric = np.sum(diffUse/np.median(diffUse))/np.sum(refUse/np.median(diffUse))
834 """Add a list of sub-band coadds together. 838 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 839 A list of coadd exposures, each exposure containing 840 the model for one subfilter. 844 coaddExposure : `lsst.afw.image.ExposureF` 845 A single coadd exposure that is the sum of the sub-bands. 847 coaddExposure = dcrCoadds[0].clone()
848 for coadd
in dcrCoadds[1:]:
849 coaddExposure.maskedImage += coadd.maskedImage
852 def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None,
853 mask=None, variance=None):
854 """Create a list of coadd exposures from a list of masked images. 858 dcrModels : `lsst.pipe.tasks.DcrModel` 859 Best fit model of the true sky after correcting chromatic effects. 860 skyInfo : `lsst.pipe.base.Struct` 861 Patch geometry information, from getSkyInfo 862 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 863 The data references to the input warped exposures. 864 weightList : `list` of `float` 865 The weight to give each input exposure in the coadd 866 calibration : `lsst.afw.Image.PhotoCalib`, optional 867 Scale factor to set the photometric calibration of an exposure. 868 coaddInputs : `lsst.afw.Image.CoaddInputs`, optional 869 A record of the observations that are included in the coadd. 870 mask : `lsst.afw.image.Mask`, optional 871 Optional mask to override the values in the final coadd. 872 variance : `lsst.afw.image.Image`, optional 873 Optional variance plane to override the values in the final coadd. 877 dcrCoadds : `list` of `lsst.afw.image.ExposureF` 878 A list of coadd exposures, each exposure containing 879 the model for one subfilter. 882 refModel = dcrModels.getReferenceImage()
883 for model
in dcrModels:
884 if self.config.accelerateModel > 1:
885 model.array = (model.array - refModel)*self.config.accelerateModel + refModel
886 coaddExposure = afwImage.ExposureF(skyInfo.bbox, skyInfo.wcs)
887 if calibration
is not None:
888 coaddExposure.setPhotoCalib(calibration)
889 if coaddInputs
is not None:
890 coaddExposure.getInfo().setCoaddInputs(coaddInputs)
894 coaddExposure.setPsf(dcrModels.psf)
895 coaddUtils.setCoaddEdgeBits(dcrModels.mask[skyInfo.bbox], dcrModels.variance[skyInfo.bbox])
896 maskedImage = afwImage.MaskedImageF(dcrModels.bbox)
897 maskedImage.image = model
898 maskedImage.mask = dcrModels.mask
899 maskedImage.variance = dcrModels.variance
900 coaddExposure.setMaskedImage(maskedImage[skyInfo.bbox])
902 coaddExposure.setMask(mask)
903 if variance
is not None:
904 coaddExposure.setVariance(variance)
905 dcrCoadds.append(coaddExposure)
909 """Calculate the gain to use for the current iteration. 911 After calculating a new DcrModel, each value is averaged with the 912 value in the corresponding pixel from the previous iteration. This 913 reduces oscillating solutions that iterative techniques are plagued by, 914 and speeds convergence. By far the biggest changes to the model 915 happen in the first couple iterations, so we can also use a more 916 aggressive gain later when the model is changing slowly. 920 convergenceList : `list` of `float` 921 The quality of fit metric from each previous iteration. 922 gainList : `list` of `float` 923 The gains used in each previous iteration: appended with the new 925 Gains are numbers between ``self.config.baseGain`` and 1. 930 Relative weight to give the new solution when updating the model. 931 A value of 1.0 gives equal weight to both solutions. 936 If ``len(convergenceList) != len(gainList)+1``. 938 nIter = len(convergenceList)
939 if nIter != len(gainList) + 1:
940 raise ValueError(
"convergenceList (%d) must be one element longer than gainList (%d)." 941 % (len(convergenceList), len(gainList)))
943 if self.config.baseGain
is None:
946 baseGain = 1./(self.config.dcrNumSubfilters - 1)
948 baseGain = self.config.baseGain
950 if self.config.useProgressiveGain
and nIter > 2:
958 estFinalConv = [((1 + gainList[i])*convergenceList[i + 1] - convergenceList[i])/gainList[i]
959 for i
in range(nIter - 1)]
962 estFinalConv = np.array(estFinalConv)
963 estFinalConv[estFinalConv < 0] = 0
965 estFinalConv = np.median(estFinalConv[max(nIter - 5, 0):])
966 lastGain = gainList[-1]
967 lastConv = convergenceList[-2]
968 newConv = convergenceList[-1]
973 predictedConv = (estFinalConv*lastGain + lastConv)/(1. + lastGain)
979 delta = (predictedConv - newConv)/((lastConv - estFinalConv)/(1 + lastGain))
980 newGain = 1 - abs(delta)
982 newGain = (newGain + lastGain)/2.
983 gain = max(baseGain, newGain)
986 gainList.append(gain)
990 """Build an array that smoothly tapers to 0 away from detected sources. 994 dcrModels : `lsst.pipe.tasks.DcrModel` 995 Best fit model of the true sky after correcting chromatic effects. 996 dcrBBox : `lsst.afw.geom.box.Box2I` 997 Sub-region of the coadd which includes a buffer to allow for DCR. 1001 weights : `numpy.ndarray` or `float` 1002 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1003 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1008 If ``useModelWeights`` is set and ``modelWeightsWidth`` is negative. 1010 if not self.config.useModelWeights:
1012 if self.config.modelWeightsWidth < 0:
1013 raise ValueError(
"modelWeightsWidth must not be negative if useModelWeights is set")
1014 convergeMask = dcrModels.mask.getPlaneBitMask(self.config.convergenceMaskPlanes)
1015 convergeMaskPixels = dcrModels.mask[dcrBBox].array & convergeMask > 0
1016 weights = np.zeros_like(dcrModels[0][dcrBBox].array)
1017 weights[convergeMaskPixels] = 1.
1018 weights = ndimage.filters.gaussian_filter(weights, self.config.modelWeightsWidth)
1019 weights /= np.max(weights)
1023 """Smoothly replace model pixel values with those from a 1024 reference at locations away from detected sources. 1028 modelImages : `list` of `lsst.afw.image.Image` 1029 The new DCR model images from the current iteration. 1030 The values will be modified in place. 1031 refImage : `lsst.afw.image.MaskedImage` 1032 A reference image used to supply the default pixel values. 1033 modelWeights : `numpy.ndarray` or `float` 1034 A 2D array of weight values that tapers smoothly to zero away from detected sources. 1035 Set to a placeholder value of 1.0 if ``self.config.useModelWeights`` is False. 1037 if self.config.useModelWeights:
1038 for model
in modelImages:
1039 model.array *= modelWeights
1040 model.array += refImage.array*(1. - modelWeights)/self.config.dcrNumSubfilters
1043 """Pre-load sub-regions of a list of exposures. 1047 bbox : `lsst.afw.geom.box.Box2I` 1049 statsCtrl : `lsst.afw.math.StatisticsControl` 1050 Statistics control object for coadd 1051 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1052 The data references to the input warped exposures. 1053 imageScalerList : `list` of `lsst.pipe.task.ImageScaler` 1054 The image scalars correct for the zero point of the exposures. 1055 spanSetMaskList : `list` of `dict` containing spanSet lists, or None 1056 Each element is dict with keys = mask plane name to add the spans to 1060 subExposures : `dict` 1061 The `dict` keys are the visit IDs, 1062 and the values are `lsst.afw.image.ExposureF` 1063 The pre-loaded exposures for the current subregion. 1064 The variance plane contains weights, and not the variance 1067 zipIterables = zip(warpRefList, imageScalerList, spanSetMaskList)
1069 for warpExpRef, imageScaler, altMaskSpans
in zipIterables:
1070 exposure = warpExpRef.get(tempExpName +
"_sub", bbox=bbox)
1071 if altMaskSpans
is not None:
1073 imageScaler.scaleMaskedImage(exposure.maskedImage)
1075 exposure.variance.array[:, :] = 0.
1077 exposure.variance.array[(exposure.mask.array & statsCtrl.getAndMask()) == 0] = 1.
1080 exposure.image.array[(exposure.mask.array & statsCtrl.getAndMask()) > 0] = 0.
1081 subExposures[warpExpRef.dataId[
"visit"]] = exposure
1085 """Compute the PSF of the coadd from the exposures with the best seeing. 1089 templateCoadd : `lsst.afw.image.ExposureF` 1090 The initial coadd exposure before accounting for DCR. 1091 warpRefList : `list` of `lsst.daf.persistence.ButlerDataRef` 1092 The data references to the input warped exposures. 1096 psf : `lsst.meas.algorithms.CoaddPsf` 1097 The average PSF of the input exposures with the best seeing. 1099 sigma2fwhm = 2.*np.sqrt(2.*np.log(2.))
1101 ccds = templateCoadd.getInfo().getCoaddInputs().ccds
1102 psfRefSize = templateCoadd.getPsf().computeShape().getDeterminantRadius()*sigma2fwhm
1104 for visitNum, warpExpRef
in enumerate(warpRefList):
1105 psf = warpExpRef.get(tempExpName).getPsf()
1106 psfSize = psf.computeShape().getDeterminantRadius()*sigma2fwhm
1107 psfSizeList.append(psfSize)
1111 sizeThreshold = min(np.median(psfSizeList), psfRefSize)
1112 goodVisits = np.array(psfSizeList) <= sizeThreshold
1113 psf = measAlg.CoaddPsf(ccds[goodVisits], templateCoadd.getWcs(),
1114 self.config.coaddPsf.makeControl())
def __init__(self, args, kwargs)
def runDataRef(self, dataRef, selectDataList=None, warpRefList=None)
def findArtifacts(self, templateCoadd, tempExpRefList, imageScalerList)
def prepareDcrInputs(self, templateCoadd, warpRefList, weightList)
def assembleMetadata(self, coaddExposure, tempExpRefList, weightList)
def calculateSingleConvergence(self, dcrModels, exposure, significanceImage, statsCtrl)
def calculateNImage(self, dcrModels, bbox, warpRefList, spanSetMaskList, statsCtrl)
def run(self, skyInfo, warpRefList, imageScalerList, weightList, supplementaryData=None)
def newModelFromResidual(self, dcrModels, residualGeneratorList, dcrBBox, statsCtrl, gain, modelWeights, refImage, dcrWeights)
def calculateConvergence(self, dcrModels, subExposures, bbox, warpRefList, weightList, statsCtrl)
def applyAltMaskPlanes(self, mask, altMaskSpans)
def getSkyInfo(self, patchRef)
Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox of the patch...
def getTempExpDatasetName(self, warpType="direct")
def prepareStats(self, mask=None)
def dcrResiduals(self, residual, visitInfo, wcs, filterInfo)
def processResults(self, coaddExposure, dataRef)
def applyModelWeights(self, modelImages, refImage, modelWeights)
def selectCoaddPsf(self, templateCoadd, warpRefList)
def loadSubExposures(self, bbox, statsCtrl, warpRefList, imageScalerList, spanSetMaskList)
def calculateGain(self, convergenceList, gainList)
def dcrAssembleSubregion(self, dcrModels, subExposures, bbox, dcrBBox, warpRefList, statsCtrl, convergenceMetric, gain, modelWeights, refImage, dcrWeights)
def _subBBoxIter(bbox, subregionSize)
def fillCoadd(self, dcrModels, skyInfo, warpRefList, weightList, calibration=None, coaddInputs=None, mask=None, variance=None)
def stackCoadd(self, dcrCoadds)
def calculateModelWeights(self, dcrModels, dcrBBox)