28from lsst.meas.algorithms
import ScaleVarianceTask
33from .
import MakeKernelTask, DecorrelateALKernelTask
34from lsst.utils.timer
import timeMethod
36__all__ = [
"AlardLuptonSubtractConfig",
"AlardLuptonSubtractTask"]
38_dimensions = (
"instrument",
"visit",
"detector")
39_defaultTemplates = {
"coaddName":
"deep",
"fakesType":
""}
43 dimensions=_dimensions,
44 defaultTemplates=_defaultTemplates):
45 template = connectionTypes.Input(
46 doc=
"Input warped template to subtract.",
47 dimensions=(
"instrument",
"visit",
"detector"),
48 storageClass=
"ExposureF",
49 name=
"{fakesType}{coaddName}Diff_templateExp"
51 science = connectionTypes.Input(
52 doc=
"Input science exposure to subtract from.",
53 dimensions=(
"instrument",
"visit",
"detector"),
54 storageClass=
"ExposureF",
55 name=
"{fakesType}calexp"
57 sources = connectionTypes.Input(
58 doc=
"Sources measured on the science exposure; "
59 "used to select sources for making the matching kernel.",
60 dimensions=(
"instrument",
"visit",
"detector"),
61 storageClass=
"SourceCatalog",
64 finalizedPsfApCorrCatalog = connectionTypes.Input(
65 doc=(
"Per-visit finalized psf models and aperture correction maps. "
66 "These catalogs use the detector id for the catalog id, "
67 "sorted on id for fast lookup."),
68 dimensions=(
"instrument",
"visit"),
69 storageClass=
"ExposureCatalog",
70 name=
"finalVisitSummary",
75 dimensions=_dimensions,
76 defaultTemplates=_defaultTemplates):
77 difference = connectionTypes.Output(
78 doc=
"Result of subtracting convolved template from science image.",
79 dimensions=(
"instrument",
"visit",
"detector"),
80 storageClass=
"ExposureF",
81 name=
"{fakesType}{coaddName}Diff_differenceTempExp",
83 matchedTemplate = connectionTypes.Output(
84 doc=
"Warped and PSF-matched template used to create `subtractedExposure`.",
85 dimensions=(
"instrument",
"visit",
"detector"),
86 storageClass=
"ExposureF",
87 name=
"{fakesType}{coaddName}Diff_matchedExp",
95 if not config.doApplyFinalizedPsf:
96 self.inputs.remove(
"finalizedPsfApCorrCatalog")
100 pipelineConnections=AlardLuptonSubtractConnections):
101 mode = lsst.pex.config.ChoiceField(
103 default=
"convolveTemplate",
104 allowed={
"auto":
"Choose which image to convolve at runtime.",
105 "convolveScience":
"Only convolve the science image.",
106 "convolveTemplate":
"Only convolve the template image."},
107 doc=
"Choose which image to convolve at runtime, or require that a specific image is convolved."
109 makeKernel = lsst.pex.config.ConfigurableField(
110 target=MakeKernelTask,
111 doc=
"Task to construct a matching kernel for convolution.",
113 doDecorrelation = lsst.pex.config.Field(
116 doc=
"Perform diffim decorrelation to undo pixel correlation due to A&L "
117 "kernel convolution? If True, also update the diffim PSF."
119 decorrelate = lsst.pex.config.ConfigurableField(
120 target=DecorrelateALKernelTask,
121 doc=
"Task to decorrelate the image difference.",
123 requiredTemplateFraction = lsst.pex.config.Field(
126 doc=
"Abort task if template covers less than this fraction of pixels."
127 " Setting to 0 will always attempt image subtraction."
129 doScaleVariance = lsst.pex.config.Field(
132 doc=
"Scale variance of the image difference?"
134 scaleVariance = lsst.pex.config.ConfigurableField(
135 target=ScaleVarianceTask,
136 doc=
"Subtask to rescale the variance of the template to the statistically expected level."
138 doSubtractBackground = lsst.pex.config.Field(
139 doc=
"Subtract the background fit when solving the kernel?",
143 doApplyFinalizedPsf = lsst.pex.config.Field(
144 doc=
"Replace science Exposure's psf and aperture correction map"
145 " with those in finalizedPsfApCorrCatalog.",
149 detectionThreshold = lsst.pex.config.Field(
152 doc=
"Minimum signal to noise ration of detected sources "
153 "to use for calculating the PSF matching kernel."
155 badSourceFlags = lsst.pex.config.ListField(
157 doc=
"Flags that, if set, the associated source should not "
158 "be used to determine the PSF matching kernel.",
159 default=(
"sky_source",
"slot_Centroid_flag",
160 "slot_ApFlux_flag",
"slot_PsfFlux_flag", ),
166 self.
makeKernel.kernel.active.spatialKernelOrder = 1
167 self.
makeKernel.kernel.active.spatialBgOrder = 2
171 """Compute the image difference of a science and template image using
172 the Alard & Lupton (1998) algorithm.
174 ConfigClass = AlardLuptonSubtractConfig
175 _DefaultName = "alardLuptonSubtract"
179 self.makeSubtask(
"decorrelate")
180 self.makeSubtask(
"makeKernel")
181 if self.config.doScaleVariance:
182 self.makeSubtask(
"scaleVariance")
190 def _applyExternalCalibrations(self, exposure, finalizedPsfApCorrCatalog):
191 """Replace calibrations (psf, and ApCorrMap) on this exposure with external ones.".
195 exposure : `lsst.afw.image.exposure.Exposure`
196 Input exposure to adjust calibrations.
198 Exposure catalog with finalized psf models
and aperture correction
199 maps to be applied
if config.doApplyFinalizedPsf=
True. Catalog uses
200 the detector id
for the catalog id, sorted on id
for fast lookup.
204 exposure : `lsst.afw.image.exposure.Exposure`
205 Exposure
with adjusted calibrations.
207 detectorId = exposure.info.getDetector().getId()
209 row = finalizedPsfApCorrCatalog.find(detectorId)
211 self.log.warning(
"Detector id %s not found in finalizedPsfApCorrCatalog; "
212 "Using original psf.", detectorId)
215 apCorrMap = row.getApCorrMap()
217 self.log.warning(
"Detector id %s has None for psf in "
218 "finalizedPsfApCorrCatalog; Using original psf and aperture correction.",
220 elif apCorrMap
is None:
221 self.log.warning(
"Detector id %s has None for apCorrMap in "
222 "finalizedPsfApCorrCatalog; Using original psf and aperture correction.",
226 exposure.info.setApCorrMap(apCorrMap)
231 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None):
232 """PSF match, subtract, and decorrelate two images.
236 template : `lsst.afw.image.ExposureF`
237 Template exposure, warped to match the science exposure.
238 science : `lsst.afw.image.ExposureF`
239 Science exposure to subtract from the template.
241 Identified sources on the science exposure. This catalog
is used to
242 select sources
in order to perform the AL PSF matching on stamp
245 Exposure catalog
with finalized psf models
and aperture correction
246 maps to be applied
if config.doApplyFinalizedPsf=
True. Catalog uses
247 the detector id
for the catalog id, sorted on id
for fast lookup.
251 results : `lsst.pipe.base.Struct`
252 ``difference`` : `lsst.afw.image.ExposureF`
253 Result of subtracting template
and science.
254 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
255 Warped
and PSF-matched template exposure.
256 ``backgroundModel`` : `lsst.afw.math.Function2D`
257 Background model that was fit
while solving
for the PSF-matching kernel
259 Kernel used to PSF-match the convolved image.
264 If an unsupported convolution mode
is supplied.
266 If there are too few sources to calculate the PSF matching kernel.
267 lsst.pipe.base.NoWorkFound
268 Raised
if fraction of good pixels, defined
as not having NO_DATA
269 set,
is less then the configured requiredTemplateFraction
272 if self.config.doApplyFinalizedPsf:
274 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
276 requiredTemplateFraction=self.config.requiredTemplateFraction)
279 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
280 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
289 templatePsfSize = getPsfFwhm(template.psf)
290 sciencePsfSize = getPsfFwhm(science.psf)
291 except InvalidParameterError:
292 self.log.info(
"Unable to evaluate PSF at the average position. "
293 "Evaluting PSF on a grid of points."
295 templatePsfSize = evaluateMeanPsfFwhm(template,
296 fwhmExposureBuffer=fwhmExposureBuffer,
297 fwhmExposureGrid=fwhmExposureGrid
299 sciencePsfSize = evaluateMeanPsfFwhm(science,
300 fwhmExposureBuffer=fwhmExposureBuffer,
301 fwhmExposureGrid=fwhmExposureGrid
303 self.log.info(
"Science PSF FWHM: %f pixels", sciencePsfSize)
304 self.log.info(
"Template PSF FWHM: %f pixels", templatePsfSize)
306 if self.config.mode ==
"auto":
307 convolveTemplate = _shapeTest(template,
309 fwhmExposureBuffer=fwhmExposureBuffer,
310 fwhmExposureGrid=fwhmExposureGrid)
312 if sciencePsfSize < templatePsfSize:
313 self.log.info(
"Average template PSF size is greater, "
314 "but science PSF greater in one dimension: convolving template image.")
316 self.log.info(
"Science PSF size is greater: convolving template image.")
318 self.log.info(
"Template PSF size is greater: convolving science image.")
319 elif self.config.mode ==
"convolveTemplate":
320 self.log.info(
"`convolveTemplate` is set: convolving template image.")
321 convolveTemplate =
True
322 elif self.config.mode ==
"convolveScience":
323 self.log.info(
"`convolveScience` is set: convolving science image.")
324 convolveTemplate =
False
326 raise RuntimeError(
"Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
328 photoCalib = template.getPhotoCalib()
329 self.log.info(
"Applying photometric calibration to template: %f", photoCalib.getCalibrationMean())
330 template.maskedImage = photoCalib.calibrateImage(template.maskedImage)
332 if self.config.doScaleVariance:
336 templateVarFactor = self.scaleVariance.
run(template.maskedImage)
337 sciVarFactor = self.scaleVariance.
run(science.maskedImage)
338 self.log.info(
"Template variance scaling factor: %.2f", templateVarFactor)
339 self.metadata.add(
"scaleTemplateVarianceFactor", templateVarFactor)
340 self.log.info(
"Science variance scaling factor: %.2f", sciVarFactor)
341 self.metadata.add(
"scaleScienceVarianceFactor", sciVarFactor)
344 self.log.info(
"%i sources used out of %i from the input catalog", len(selectSources), len(sources))
345 if len(selectSources) < self.config.makeKernel.nStarPerCell:
346 self.log.warning(
"Too few sources to calculate the PSF matching kernel: "
347 "%i selected but %i needed for the calculation.",
348 len(selectSources), self.config.makeKernel.nStarPerCell)
349 raise RuntimeError(
"Cannot compute PSF matching kernel: too few sources selected.")
355 return subtractResults
358 """Convolve the template image with a PSF-matching kernel and subtract
359 from the science image.
363 template : `lsst.afw.image.ExposureF`
364 Template exposure, warped to match the science exposure.
365 science : `lsst.afw.image.ExposureF`
366 Science exposure to subtract
from the template.
368 Identified sources on the science exposure. This catalog
is used to
369 select sources
in order to perform the AL PSF matching on stamp
374 results : `lsst.pipe.base.Struct`
376 ``difference`` : `lsst.afw.image.ExposureF`
377 Result of subtracting template
and science.
378 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
379 Warped
and PSF-matched template exposure.
380 ``backgroundModel`` : `lsst.afw.math.Function2D`
381 Background model that was fit
while solving
for the PSF-matching kernel
383 Kernel used to PSF-match the template to the science image.
385 kernelSources = self.makeKernel.selectKernelSources(template, science,
386 candidateList=selectSources,
388 kernelResult = self.makeKernel.
run(template, science, kernelSources,
391 matchedTemplate = self.
_convolveExposure(template, kernelResult.psfMatchingKernel,
393 bbox=science.getBBox(),
395 photoCalib=science.getPhotoCalib())
396 difference = _subtractImages(science, matchedTemplate,
397 backgroundModel=(kernelResult.backgroundModel
398 if self.config.doSubtractBackground
else None))
399 correctedExposure = self.
finalize(template, science, difference, kernelResult.psfMatchingKernel,
400 templateMatched=
True)
402 return lsst.pipe.base.Struct(difference=correctedExposure,
403 matchedTemplate=matchedTemplate,
404 matchedScience=science,
405 backgroundModel=kernelResult.backgroundModel,
406 psfMatchingKernel=kernelResult.psfMatchingKernel)
409 """Convolve the science image with a PSF-matching kernel and subtract the template image.
413 template : `lsst.afw.image.ExposureF`
414 Template exposure, warped to match the science exposure.
415 science : `lsst.afw.image.ExposureF`
416 Science exposure to subtract from the template.
418 Identified sources on the science exposure. This catalog
is used to
419 select sources
in order to perform the AL PSF matching on stamp
424 results : `lsst.pipe.base.Struct`
426 ``difference`` : `lsst.afw.image.ExposureF`
427 Result of subtracting template
and science.
428 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
429 Warped template exposure. Note that
in this case, the template
430 is not PSF-matched to the science image.
431 ``backgroundModel`` : `lsst.afw.math.Function2D`
432 Background model that was fit
while solving
for the PSF-matching kernel
434 Kernel used to PSF-match the science image to the template.
436 kernelSources = self.makeKernel.selectKernelSources(science, template,
437 candidateList=selectSources,
439 kernelResult = self.makeKernel.
run(science, template, kernelSources,
441 modelParams = kernelResult.backgroundModel.getParameters()
443 kernelResult.backgroundModel.setParameters([-p
for p
in modelParams])
445 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
446 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=
False)
453 matchedScience.maskedImage /= norm
454 matchedTemplate = template.clone()[science.getBBox()]
455 matchedTemplate.maskedImage /= norm
456 matchedTemplate.setPhotoCalib(science.getPhotoCalib())
458 difference = _subtractImages(matchedScience, matchedTemplate,
459 backgroundModel=(kernelResult.backgroundModel
460 if self.config.doSubtractBackground
else None))
462 correctedExposure = self.
finalize(template, science, difference, kernelResult.psfMatchingKernel,
463 templateMatched=
False)
465 return lsst.pipe.base.Struct(difference=correctedExposure,
466 matchedTemplate=matchedTemplate,
467 matchedScience=matchedScience,
468 backgroundModel=kernelResult.backgroundModel,
469 psfMatchingKernel=kernelResult.psfMatchingKernel,)
471 def finalize(self, template, science, difference, kernel,
472 templateMatched=True,
475 spatiallyVarying=False):
476 """Decorrelate the difference image to undo the noise correlations
477 caused by convolution.
481 template : `lsst.afw.image.ExposureF`
482 Template exposure, warped to match the science exposure.
483 science : `lsst.afw.image.ExposureF`
484 Science exposure to subtract from the template.
485 difference : `lsst.afw.image.ExposureF`
486 Result of subtracting template
and science.
488 An (optionally spatially-varying) PSF matching kernel
489 templateMatched : `bool`, optional
490 Was the template PSF-matched to the science image?
491 preConvMode : `bool`, optional
492 Was the science image preconvolved
with its own PSF
493 before PSF matching the template?
495 If
not `
None`, then the science image was pre-convolved
with
496 (the reflection of) this kernel. Must be normalized to sum to 1.
497 spatiallyVarying : `bool`, optional
498 Compute the decorrelation kernel spatially varying across the image?
502 correctedExposure : `lsst.afw.image.ExposureF`
503 The decorrelated image difference.
507 mask = difference.mask
508 mask &= ~(mask.getPlaneBitMask(
"DETECTED") | mask.getPlaneBitMask(
"DETECTED_NEGATIVE"))
510 if self.config.doDecorrelation:
511 self.log.info(
"Decorrelating image difference.")
512 correctedExposure = self.decorrelate.
run(science, template[science.getBBox()], difference, kernel,
513 templateMatched=templateMatched,
514 preConvMode=preConvMode,
515 preConvKernel=preConvKernel,
516 spatiallyVarying=spatiallyVarying).correctedExposure
518 self.log.info(
"NOT decorrelating image difference.")
519 correctedExposure = difference
520 return correctedExposure
523 def _validateExposures(template, science):
524 """Check that the WCS of the two Exposures match, and the template bbox
525 contains the science bbox.
529 template : `lsst.afw.image.ExposureF`
530 Template exposure, warped to match the science exposure.
531 science : `lsst.afw.image.ExposureF`
532 Science exposure to subtract from the template.
537 Raised
if the WCS of the template
is not equal to the science WCS,
538 or if the science image
is not fully contained
in the template
541 assert template.wcs == science.wcs,\
542 "Template and science exposure WCS are not identical."
543 templateBBox = template.getBBox()
544 scienceBBox = science.getBBox()
546 assert templateBBox.contains(scienceBBox),\
547 "Template bbox does not contain all of the science image."
550 def _convolveExposure(exposure, kernel, convolutionControl,
554 """Convolve an exposure with the given kernel.
558 exposure : `lsst.afw.Exposure`
559 exposure to convolve.
561 PSF matching kernel computed in the ``makeKernel`` subtask.
563 Configuration
for convolve algorithm.
565 Bounding box to trim the convolved exposure to.
567 Point spread function (PSF) to set
for the convolved exposure.
569 Photometric calibration of the convolved exposure.
573 convolvedExp : `lsst.afw.Exposure`
576 convolvedExposure = exposure.clone()
578 convolvedExposure.setPsf(psf)
579 if photoCalib
is not None:
580 convolvedExposure.setPhotoCalib(photoCalib)
581 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox())
583 convolvedExposure.setMaskedImage(convolvedImage)
585 return convolvedExposure
587 return convolvedExposure[bbox]
589 def _sourceSelector(self, sources):
590 """Select sources from a catalog that meet the selection criteria.
595 Input source catalog to select sources from.
600 The source catalog filtered to include only the selected sources.
602 flags = [True, ]*len(sources)
603 for flag
in self.config.badSourceFlags:
605 flags *= ~sources[flag]
606 except Exception
as e:
607 self.log.warning(
"Could not apply source flag: %s", e)
608 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
610 selectSources = sources[flags]
612 return selectSources.copy(deep=
True)
616 """Raise NoWorkFound if template coverage < requiredTemplateFraction
620 templateExposure : `lsst.afw.image.ExposureF`
621 The template exposure to check
623 Logger for printing output.
624 requiredTemplateFraction : `float`, optional
625 Fraction of pixels of the science image required to have coverage
630 lsst.pipe.base.NoWorkFound
631 Raised
if fraction of good pixels, defined
as not having NO_DATA
632 set,
is less then the configured requiredTemplateFraction
636 pixNoData = np.count_nonzero(templateExposure.mask.array
637 & templateExposure.mask.getPlaneBitMask(
'NO_DATA'))
638 pixGood = templateExposure.getBBox().getArea() - pixNoData
639 logger.info(
"template has %d good pixels (%.1f%%)", pixGood,
640 100*pixGood/templateExposure.getBBox().getArea())
642 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
643 message = (
"Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
644 "To force subtraction, set config requiredTemplateFraction=0." % (
645 100*pixGood/templateExposure.getBBox().getArea(),
646 100*requiredTemplateFraction))
647 raise lsst.pipe.base.NoWorkFound(message)
650def _subtractImages(science, template, backgroundModel=None):
651 """Subtract template from science, propagating relevant metadata.
655 science : `lsst.afw.Exposure`
656 The input science image.
657 template : `lsst.afw.Exposure`
658 The template to subtract from the science image.
659 backgroundModel : `lsst.afw.MaskedImage`, optional
660 Differential background model
664 difference : `lsst.afw.Exposure`
665 The subtracted image.
667 difference = science.clone()
668 if backgroundModel
is not None:
669 difference.maskedImage -= backgroundModel
670 difference.maskedImage -= template.maskedImage
674def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
675 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
680 Exposure with the reference point spread function (PSF) to evaluate.
682 Exposure
with a candidate point spread function (PSF) to evaluate.
683 fwhmExposureBuffer : `float`
684 Fractional buffer margin to be left out of all sides of the image
685 during the construction of the grid to compute mean PSF FWHM
in an
686 exposure,
if the PSF
is not available at its average position.
687 fwhmExposureGrid : `int`
688 Grid size to compute the mean FWHM
in an exposure,
if the PSF
is not
689 available at its average position.
693 True if ``exp1`` has a PSF that
is not wider than that of ``exp2``
in
697 shape1 = getPsfFwhm(exp1.psf, average=
False)
698 shape2 = getPsfFwhm(exp2.psf, average=
False)
699 except InvalidParameterError:
700 shape1 = evaluateMeanPsfFwhm(exp1,
701 fwhmExposureBuffer=fwhmExposureBuffer,
702 fwhmExposureGrid=fwhmExposureGrid
704 shape2 = evaluateMeanPsfFwhm(exp2,
705 fwhmExposureBuffer=fwhmExposureBuffer,
706 fwhmExposureGrid=fwhmExposureGrid
708 return shape1 <= shape2
711 xTest = shape1[0] <= shape2[0]
712 yTest = shape1[1] <= shape2[1]
def __init__(self, *config=None)
def finalize(self, template, science, difference, kernel, templateMatched=True, preConvMode=False, preConvKernel=None, spatiallyVarying=False)
def _convolveExposure(exposure, kernel, convolutionControl, bbox=None, psf=None, photoCalib=None)
def runConvolveTemplate(self, template, science, selectSources)
def _sourceSelector(self, sources)
def _applyExternalCalibrations(self, exposure, finalizedPsfApCorrCatalog)
def _validateExposures(template, science)
def __init__(self, **kwargs)
def run(self, template, science, sources, finalizedPsfApCorrCatalog=None)
def runConvolveScience(self, template, science, selectSources)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, ConvolutionControl const &convolutionControl=ConvolutionControl())
def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.)