Coverage for python/lsst/ip/diffim/subtractImages.py: 25%
280 statements
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-21 18:29 +0000
« prev ^ index » next coverage.py v7.3.1, created at 2023-09-21 18:29 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import warnings
24import numpy as np
26import lsst.afw.image
27import lsst.afw.math
28import lsst.geom
29from lsst.utils.introspection import find_outside_stacklevel
30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm
31from lsst.meas.algorithms import ScaleVarianceTask
32import lsst.pex.config
33import lsst.pipe.base
34from lsst.pex.exceptions import InvalidParameterError
35from lsst.pipe.base import connectionTypes
36from . import MakeKernelTask, DecorrelateALKernelTask
37from lsst.utils.timer import timeMethod
39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask",
40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"]
42_dimensions = ("instrument", "visit", "detector")
43_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
47 dimensions=_dimensions,
48 defaultTemplates=_defaultTemplates):
49 template = connectionTypes.Input(
50 doc="Input warped template to subtract.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}{coaddName}Diff_templateExp"
54 )
55 science = connectionTypes.Input(
56 doc="Input science exposure to subtract from.",
57 dimensions=("instrument", "visit", "detector"),
58 storageClass="ExposureF",
59 name="{fakesType}calexp"
60 )
61 sources = connectionTypes.Input(
62 doc="Sources measured on the science exposure; "
63 "used to select sources for making the matching kernel.",
64 dimensions=("instrument", "visit", "detector"),
65 storageClass="SourceCatalog",
66 name="{fakesType}src"
67 )
68 finalizedPsfApCorrCatalog = connectionTypes.Input(
69 doc=("Per-visit finalized psf models and aperture correction maps. "
70 "These catalogs use the detector id for the catalog id, "
71 "sorted on id for fast lookup."),
72 dimensions=("instrument", "visit"),
73 storageClass="ExposureCatalog",
74 name="finalVisitSummary",
75 # TODO: remove on DM-39854.
76 deprecated=(
77 "Deprecated in favor of visitSummary. Will be removed after v26."
78 )
79 )
80 visitSummary = connectionTypes.Input(
81 doc=("Per-visit catalog with final calibration objects. "
82 "These catalogs use the detector id for the catalog id, "
83 "sorted on id for fast lookup."),
84 dimensions=("instrument", "visit"),
85 storageClass="ExposureCatalog",
86 name="finalVisitSummary",
87 )
89 def __init__(self, *, config=None):
90 super().__init__(config=config)
91 if not config.doApplyFinalizedPsf:
92 self.inputs.remove("finalizedPsfApCorrCatalog")
93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf:
94 del self.visitSummary
97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
98 dimensions=_dimensions,
99 defaultTemplates=_defaultTemplates):
100 difference = connectionTypes.Output(
101 doc="Result of subtracting convolved template from science image.",
102 dimensions=("instrument", "visit", "detector"),
103 storageClass="ExposureF",
104 name="{fakesType}{coaddName}Diff_differenceTempExp",
105 )
106 matchedTemplate = connectionTypes.Output(
107 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
108 dimensions=("instrument", "visit", "detector"),
109 storageClass="ExposureF",
110 name="{fakesType}{coaddName}Diff_matchedExp",
111 )
114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections,
115 dimensions=_dimensions,
116 defaultTemplates=_defaultTemplates):
117 scoreExposure = connectionTypes.Output(
118 doc="The maximum likelihood image, used for the detection of diaSources.",
119 dimensions=("instrument", "visit", "detector"),
120 storageClass="ExposureF",
121 name="{fakesType}{coaddName}Diff_scoreExp",
122 )
125class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections):
126 pass
129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
130 makeKernel = lsst.pex.config.ConfigurableField(
131 target=MakeKernelTask,
132 doc="Task to construct a matching kernel for convolution.",
133 )
134 doDecorrelation = lsst.pex.config.Field(
135 dtype=bool,
136 default=True,
137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
138 "kernel convolution? If True, also update the diffim PSF."
139 )
140 decorrelate = lsst.pex.config.ConfigurableField(
141 target=DecorrelateALKernelTask,
142 doc="Task to decorrelate the image difference.",
143 )
144 requiredTemplateFraction = lsst.pex.config.Field(
145 dtype=float,
146 default=0.1,
147 doc="Abort task if template covers less than this fraction of pixels."
148 " Setting to 0 will always attempt image subtraction."
149 )
150 doScaleVariance = lsst.pex.config.Field(
151 dtype=bool,
152 default=True,
153 doc="Scale variance of the image difference?"
154 )
155 scaleVariance = lsst.pex.config.ConfigurableField(
156 target=ScaleVarianceTask,
157 doc="Subtask to rescale the variance of the template to the statistically expected level."
158 )
159 doSubtractBackground = lsst.pex.config.Field(
160 doc="Subtract the background fit when solving the kernel?",
161 dtype=bool,
162 default=True,
163 )
164 doApplyFinalizedPsf = lsst.pex.config.Field(
165 doc="Replace science Exposure's psf and aperture correction map"
166 " with those in finalizedPsfApCorrCatalog.",
167 dtype=bool,
168 default=False,
169 # TODO: remove on DM-39854.
170 deprecated=(
171 "Deprecated in favor of doApplyExternalCalibrations. "
172 "Will be removed after v26."
173 )
174 )
175 doApplyExternalCalibrations = lsst.pex.config.Field(
176 doc=(
177 "Replace science Exposure's calibration objects with those"
178 " in visitSummary. Ignored if `doApplyFinalizedPsf is True."
179 ),
180 dtype=bool,
181 default=False,
182 )
183 detectionThreshold = lsst.pex.config.Field(
184 dtype=float,
185 default=10,
186 doc="Minimum signal to noise ratio of detected sources "
187 "to use for calculating the PSF matching kernel."
188 )
189 badSourceFlags = lsst.pex.config.ListField(
190 dtype=str,
191 doc="Flags that, if set, the associated source should not "
192 "be used to determine the PSF matching kernel.",
193 default=("sky_source", "slot_Centroid_flag",
194 "slot_ApFlux_flag", "slot_PsfFlux_flag", ),
195 )
196 badMaskPlanes = lsst.pex.config.ListField(
197 dtype=str,
198 default=("NO_DATA", "BAD", "SAT", "EDGE"),
199 doc="Mask planes to exclude when selecting sources for PSF matching."
200 )
201 preserveTemplateMask = lsst.pex.config.ListField(
202 dtype=str,
203 default=("NO_DATA", "BAD", "SAT", "INJECTED", "INJECTED_CORE"),
204 doc="Mask planes from the template to propagate to the image difference."
205 )
207 def setDefaults(self):
208 self.makeKernel.kernel.name = "AL"
209 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground
210 self.makeKernel.kernel.active.spatialKernelOrder = 1
211 self.makeKernel.kernel.active.spatialBgOrder = 2
214class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
215 pipelineConnections=AlardLuptonSubtractConnections):
216 mode = lsst.pex.config.ChoiceField(
217 dtype=str,
218 default="convolveTemplate",
219 allowed={"auto": "Choose which image to convolve at runtime.",
220 "convolveScience": "Only convolve the science image.",
221 "convolveTemplate": "Only convolve the template image."},
222 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
223 )
226class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
227 """Compute the image difference of a science and template image using
228 the Alard & Lupton (1998) algorithm.
229 """
230 ConfigClass = AlardLuptonSubtractConfig
231 _DefaultName = "alardLuptonSubtract"
233 def __init__(self, **kwargs):
234 super().__init__(**kwargs)
235 self.makeSubtask("decorrelate")
236 self.makeSubtask("makeKernel")
237 if self.config.doScaleVariance:
238 self.makeSubtask("scaleVariance")
240 self.convolutionControl = lsst.afw.math.ConvolutionControl()
241 # Normalization is an extra, unnecessary, calculation and will result
242 # in mis-subtraction of the images if there are calibration errors.
243 self.convolutionControl.setDoNormalize(False)
244 self.convolutionControl.setDoCopyEdge(True)
246 def _applyExternalCalibrations(self, exposure, visitSummary):
247 """Replace calibrations (psf, and ApCorrMap) on this exposure with
248 external ones.".
250 Parameters
251 ----------
252 exposure : `lsst.afw.image.exposure.Exposure`
253 Input exposure to adjust calibrations.
254 visitSummary : `lsst.afw.table.ExposureCatalog`
255 Exposure catalog with external calibrations to be applied. Catalog
256 uses the detector id for the catalog id, sorted on id for fast
257 lookup.
259 Returns
260 -------
261 exposure : `lsst.afw.image.exposure.Exposure`
262 Exposure with adjusted calibrations.
263 """
264 detectorId = exposure.info.getDetector().getId()
266 row = visitSummary.find(detectorId)
267 if row is None:
268 self.log.warning("Detector id %s not found in external calibrations catalog; "
269 "Using original calibrations.", detectorId)
270 else:
271 psf = row.getPsf()
272 apCorrMap = row.getApCorrMap()
273 if psf is None:
274 self.log.warning("Detector id %s has None for psf in "
275 "external calibrations catalog; Using original psf and aperture correction.",
276 detectorId)
277 elif apCorrMap is None:
278 self.log.warning("Detector id %s has None for apCorrMap in "
279 "external calibrations catalog; Using original psf and aperture correction.",
280 detectorId)
281 else:
282 exposure.setPsf(psf)
283 exposure.info.setApCorrMap(apCorrMap)
285 return exposure
287 @timeMethod
288 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None,
289 visitSummary=None):
290 """PSF match, subtract, and decorrelate two images.
292 Parameters
293 ----------
294 template : `lsst.afw.image.ExposureF`
295 Template exposure, warped to match the science exposure.
296 science : `lsst.afw.image.ExposureF`
297 Science exposure to subtract from the template.
298 sources : `lsst.afw.table.SourceCatalog`
299 Identified sources on the science exposure. This catalog is used to
300 select sources in order to perform the AL PSF matching on stamp
301 images around them.
302 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
303 Exposure catalog with finalized psf models and aperture correction
304 maps to be applied. Catalog uses the detector id for the catalog
305 id, sorted on id for fast lookup. Deprecated in favor of
306 ``visitSummary``, and will be removed after v26.
307 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
308 Exposure catalog with external calibrations to be applied. Catalog
309 uses the detector id for the catalog id, sorted on id for fast
310 lookup. Ignored (for temporary backwards compatibility) if
311 ``finalizedPsfApCorrCatalog`` is provided.
313 Returns
314 -------
315 results : `lsst.pipe.base.Struct`
316 ``difference`` : `lsst.afw.image.ExposureF`
317 Result of subtracting template and science.
318 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
319 Warped and PSF-matched template exposure.
320 ``backgroundModel`` : `lsst.afw.math.Function2D`
321 Background model that was fit while solving for the
322 PSF-matching kernel
323 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
324 Kernel used to PSF-match the convolved image.
326 Raises
327 ------
328 RuntimeError
329 If an unsupported convolution mode is supplied.
330 RuntimeError
331 If there are too few sources to calculate the PSF matching kernel.
332 lsst.pipe.base.NoWorkFound
333 Raised if fraction of good pixels, defined as not having NO_DATA
334 set, is less then the configured requiredTemplateFraction
335 """
337 if finalizedPsfApCorrCatalog is not None:
338 warnings.warn(
339 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
340 "argument, and will be removed after v26.",
341 FutureWarning,
342 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
343 )
344 visitSummary = finalizedPsfApCorrCatalog
346 self._prepareInputs(template, science, visitSummary=visitSummary)
348 # In the event that getPsfFwhm fails, evaluate the PSF on a grid.
349 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
350 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
352 # Calling getPsfFwhm on template.psf fails on some rare occasions when
353 # the template has no input exposures at the average position of the
354 # stars. So we try getPsfFwhm first on template, and if that fails we
355 # evaluate the PSF on a grid specified by fwhmExposure* fields.
356 # To keep consistent definitions for PSF size on the template and
357 # science images, we use the same method for both.
358 try:
359 templatePsfSize = getPsfFwhm(template.psf)
360 sciencePsfSize = getPsfFwhm(science.psf)
361 except InvalidParameterError:
362 self.log.info("Unable to evaluate PSF at the average position. "
363 "Evaluting PSF on a grid of points."
364 )
365 templatePsfSize = evaluateMeanPsfFwhm(template,
366 fwhmExposureBuffer=fwhmExposureBuffer,
367 fwhmExposureGrid=fwhmExposureGrid
368 )
369 sciencePsfSize = evaluateMeanPsfFwhm(science,
370 fwhmExposureBuffer=fwhmExposureBuffer,
371 fwhmExposureGrid=fwhmExposureGrid
372 )
373 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize)
374 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize)
375 self.metadata.add("sciencePsfSize", sciencePsfSize)
376 self.metadata.add("templatePsfSize", templatePsfSize)
377 selectSources = self._sourceSelector(sources, science.mask)
379 if self.config.mode == "auto":
380 convolveTemplate = _shapeTest(template,
381 science,
382 fwhmExposureBuffer=fwhmExposureBuffer,
383 fwhmExposureGrid=fwhmExposureGrid)
384 if convolveTemplate:
385 if sciencePsfSize < templatePsfSize:
386 self.log.info("Average template PSF size is greater, "
387 "but science PSF greater in one dimension: convolving template image.")
388 else:
389 self.log.info("Science PSF size is greater: convolving template image.")
390 else:
391 self.log.info("Template PSF size is greater: convolving science image.")
392 elif self.config.mode == "convolveTemplate":
393 self.log.info("`convolveTemplate` is set: convolving template image.")
394 convolveTemplate = True
395 elif self.config.mode == "convolveScience":
396 self.log.info("`convolveScience` is set: convolving science image.")
397 convolveTemplate = False
398 else:
399 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
401 if convolveTemplate:
402 self.metadata.add("convolvedExposure", "Template")
403 subtractResults = self.runConvolveTemplate(template, science, selectSources)
404 else:
405 self.metadata.add("convolvedExposure", "Science")
406 subtractResults = self.runConvolveScience(template, science, selectSources)
408 return subtractResults
410 def runConvolveTemplate(self, template, science, selectSources):
411 """Convolve the template image with a PSF-matching kernel and subtract
412 from the science image.
414 Parameters
415 ----------
416 template : `lsst.afw.image.ExposureF`
417 Template exposure, warped to match the science exposure.
418 science : `lsst.afw.image.ExposureF`
419 Science exposure to subtract from the template.
420 selectSources : `lsst.afw.table.SourceCatalog`
421 Identified sources on the science exposure. This catalog is used to
422 select sources in order to perform the AL PSF matching on stamp
423 images around them.
425 Returns
426 -------
427 results : `lsst.pipe.base.Struct`
429 ``difference`` : `lsst.afw.image.ExposureF`
430 Result of subtracting template and science.
431 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
432 Warped and PSF-matched template exposure.
433 ``backgroundModel`` : `lsst.afw.math.Function2D`
434 Background model that was fit while solving for the PSF-matching kernel
435 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
436 Kernel used to PSF-match the template to the science image.
437 """
438 kernelSources = self.makeKernel.selectKernelSources(template, science,
439 candidateList=selectSources,
440 preconvolved=False)
441 kernelResult = self.makeKernel.run(template, science, kernelSources,
442 preconvolved=False)
444 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
445 self.convolutionControl,
446 bbox=science.getBBox(),
447 psf=science.psf,
448 photoCalib=science.photoCalib)
450 difference = _subtractImages(science, matchedTemplate,
451 backgroundModel=(kernelResult.backgroundModel
452 if self.config.doSubtractBackground else None))
453 correctedExposure = self.finalize(template, science, difference,
454 kernelResult.psfMatchingKernel,
455 templateMatched=True)
457 return lsst.pipe.base.Struct(difference=correctedExposure,
458 matchedTemplate=matchedTemplate,
459 matchedScience=science,
460 backgroundModel=kernelResult.backgroundModel,
461 psfMatchingKernel=kernelResult.psfMatchingKernel)
463 def runConvolveScience(self, template, science, selectSources):
464 """Convolve the science image with a PSF-matching kernel and subtract the template image.
466 Parameters
467 ----------
468 template : `lsst.afw.image.ExposureF`
469 Template exposure, warped to match the science exposure.
470 science : `lsst.afw.image.ExposureF`
471 Science exposure to subtract from the template.
472 selectSources : `lsst.afw.table.SourceCatalog`
473 Identified sources on the science exposure. This catalog is used to
474 select sources in order to perform the AL PSF matching on stamp
475 images around them.
477 Returns
478 -------
479 results : `lsst.pipe.base.Struct`
481 ``difference`` : `lsst.afw.image.ExposureF`
482 Result of subtracting template and science.
483 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
484 Warped template exposure. Note that in this case, the template
485 is not PSF-matched to the science image.
486 ``backgroundModel`` : `lsst.afw.math.Function2D`
487 Background model that was fit while solving for the PSF-matching kernel
488 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
489 Kernel used to PSF-match the science image to the template.
490 """
491 bbox = science.getBBox()
492 kernelSources = self.makeKernel.selectKernelSources(science, template,
493 candidateList=selectSources,
494 preconvolved=False)
495 kernelResult = self.makeKernel.run(science, template, kernelSources,
496 preconvolved=False)
497 modelParams = kernelResult.backgroundModel.getParameters()
498 # We must invert the background model if the matching kernel is solved for the science image.
499 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
501 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
502 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False)
504 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel,
505 self.convolutionControl,
506 psf=template.psf)
508 # Place back on native photometric scale
509 matchedScience.maskedImage /= norm
510 matchedTemplate = template.clone()[bbox]
511 matchedTemplate.maskedImage /= norm
512 matchedTemplate.setPhotoCalib(science.photoCalib)
514 difference = _subtractImages(matchedScience, matchedTemplate,
515 backgroundModel=(kernelResult.backgroundModel
516 if self.config.doSubtractBackground else None))
518 correctedExposure = self.finalize(template, science, difference,
519 kernelResult.psfMatchingKernel,
520 templateMatched=False)
522 return lsst.pipe.base.Struct(difference=correctedExposure,
523 matchedTemplate=matchedTemplate,
524 matchedScience=matchedScience,
525 backgroundModel=kernelResult.backgroundModel,
526 psfMatchingKernel=kernelResult.psfMatchingKernel,)
528 def finalize(self, template, science, difference, kernel,
529 templateMatched=True,
530 preConvMode=False,
531 preConvKernel=None,
532 spatiallyVarying=False):
533 """Decorrelate the difference image to undo the noise correlations
534 caused by convolution.
536 Parameters
537 ----------
538 template : `lsst.afw.image.ExposureF`
539 Template exposure, warped to match the science exposure.
540 science : `lsst.afw.image.ExposureF`
541 Science exposure to subtract from the template.
542 difference : `lsst.afw.image.ExposureF`
543 Result of subtracting template and science.
544 kernel : `lsst.afw.math.Kernel`
545 An (optionally spatially-varying) PSF matching kernel
546 templateMatched : `bool`, optional
547 Was the template PSF-matched to the science image?
548 preConvMode : `bool`, optional
549 Was the science image preconvolved with its own PSF
550 before PSF matching the template?
551 preConvKernel : `lsst.afw.detection.Psf`, optional
552 If not `None`, then the science image was pre-convolved with
553 (the reflection of) this kernel. Must be normalized to sum to 1.
554 spatiallyVarying : `bool`, optional
555 Compute the decorrelation kernel spatially varying across the image?
557 Returns
558 -------
559 correctedExposure : `lsst.afw.image.ExposureF`
560 The decorrelated image difference.
561 """
562 # Erase existing detection mask planes.
563 # We don't want the detection mask from the science image
564 mask = difference.mask
565 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
567 # We have cleared the template mask plane, so copy the mask plane of
568 # the image difference so that we can calculate correct statistics
569 # during decorrelation. Do this regardless of whether decorrelation is
570 # used for consistency.
571 template[science.getBBox()].mask.array[...] = difference.mask.array[...]
572 if self.config.doDecorrelation:
573 self.log.info("Decorrelating image difference.")
574 # We have cleared the template mask plane, so copy the mask plane of
575 # the image difference so that we can calculate correct statistics
576 # during decorrelation
577 template[science.getBBox()].mask.array[...] = difference.mask.array[...]
578 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
579 templateMatched=templateMatched,
580 preConvMode=preConvMode,
581 preConvKernel=preConvKernel,
582 spatiallyVarying=spatiallyVarying).correctedExposure
583 else:
584 self.log.info("NOT decorrelating image difference.")
585 correctedExposure = difference
586 return correctedExposure
588 @staticmethod
589 def _validateExposures(template, science):
590 """Check that the WCS of the two Exposures match, and the template bbox
591 contains the science bbox.
593 Parameters
594 ----------
595 template : `lsst.afw.image.ExposureF`
596 Template exposure, warped to match the science exposure.
597 science : `lsst.afw.image.ExposureF`
598 Science exposure to subtract from the template.
600 Raises
601 ------
602 AssertionError
603 Raised if the WCS of the template is not equal to the science WCS,
604 or if the science image is not fully contained in the template
605 bounding box.
606 """
607 assert template.wcs == science.wcs,\
608 "Template and science exposure WCS are not identical."
609 templateBBox = template.getBBox()
610 scienceBBox = science.getBBox()
612 assert templateBBox.contains(scienceBBox),\
613 "Template bbox does not contain all of the science image."
615 def _convolveExposure(self, exposure, kernel, convolutionControl,
616 bbox=None,
617 psf=None,
618 photoCalib=None,
619 interpolateBadMaskPlanes=False,
620 ):
621 """Convolve an exposure with the given kernel.
623 Parameters
624 ----------
625 exposure : `lsst.afw.Exposure`
626 exposure to convolve.
627 kernel : `lsst.afw.math.LinearCombinationKernel`
628 PSF matching kernel computed in the ``makeKernel`` subtask.
629 convolutionControl : `lsst.afw.math.ConvolutionControl`
630 Configuration for convolve algorithm.
631 bbox : `lsst.geom.Box2I`, optional
632 Bounding box to trim the convolved exposure to.
633 psf : `lsst.afw.detection.Psf`, optional
634 Point spread function (PSF) to set for the convolved exposure.
635 photoCalib : `lsst.afw.image.PhotoCalib`, optional
636 Photometric calibration of the convolved exposure.
638 Returns
639 -------
640 convolvedExp : `lsst.afw.Exposure`
641 The convolved image.
642 """
643 convolvedExposure = exposure.clone()
644 if psf is not None:
645 convolvedExposure.setPsf(psf)
646 if photoCalib is not None:
647 convolvedExposure.setPhotoCalib(photoCalib)
648 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None:
649 nInterp = _interpolateImage(convolvedExposure.maskedImage,
650 self.config.badMaskPlanes)
651 self.metadata.add("nInterpolated", nInterp)
652 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox())
653 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl)
654 convolvedExposure.setMaskedImage(convolvedImage)
655 if bbox is None:
656 return convolvedExposure
657 else:
658 return convolvedExposure[bbox]
660 def _sourceSelector(self, sources, mask):
661 """Select sources from a catalog that meet the selection criteria.
663 Parameters
664 ----------
665 sources : `lsst.afw.table.SourceCatalog`
666 Input source catalog to select sources from.
667 mask : `lsst.afw.image.Mask`
668 The image mask plane to use to reject sources
669 based on their location on the ccd.
671 Returns
672 -------
673 selectSources : `lsst.afw.table.SourceCatalog`
674 The input source catalog, with flagged and low signal-to-noise
675 sources removed.
677 Raises
678 ------
679 RuntimeError
680 If there are too few sources to compute the PSF matching kernel
681 remaining after source selection.
682 """
683 flags = np.ones(len(sources), dtype=bool)
684 for flag in self.config.badSourceFlags:
685 try:
686 flags *= ~sources[flag]
687 except Exception as e:
688 self.log.warning("Could not apply source flag: %s", e)
689 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
690 flags *= sToNFlag
691 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes)
692 selectSources = sources[flags]
693 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog",
694 len(selectSources), len(sources), 100*len(selectSources)/len(sources))
695 if len(selectSources) < self.config.makeKernel.nStarPerCell:
696 self.log.error("Too few sources to calculate the PSF matching kernel: "
697 "%i selected but %i needed for the calculation.",
698 len(selectSources), self.config.makeKernel.nStarPerCell)
699 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.")
700 self.metadata.add("nPsfSources", len(selectSources))
702 return selectSources.copy(deep=True)
704 @staticmethod
705 def _checkMask(mask, sources, badMaskPlanes):
706 """Exclude sources that are located on masked pixels.
708 Parameters
709 ----------
710 mask : `lsst.afw.image.Mask`
711 The image mask plane to use to reject sources
712 based on the location of their centroid on the ccd.
713 sources : `lsst.afw.table.SourceCatalog`
714 The source catalog to evaluate.
715 badMaskPlanes : `list` of `str`
716 List of the names of the mask planes to exclude.
718 Returns
719 -------
720 flags : `numpy.ndarray` of `bool`
721 Array indicating whether each source in the catalog should be
722 kept (True) or rejected (False) based on the value of the
723 mask plane at its location.
724 """
725 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(badMaskPlanes)
726 xv = np.rint(sources.getX() - mask.getX0())
727 yv = np.rint(sources.getY() - mask.getY0())
729 mv = mask.array[yv.astype(int), xv.astype(int)]
730 flags = np.bitwise_and(mv, badPixelMask) == 0
731 return flags
733 def _prepareInputs(self, template, science, visitSummary=None):
734 """Perform preparatory calculations common to all Alard&Lupton Tasks.
736 Parameters
737 ----------
738 template : `lsst.afw.image.ExposureF`
739 Template exposure, warped to match the science exposure. The
740 variance plane of the template image is modified in place.
741 science : `lsst.afw.image.ExposureF`
742 Science exposure to subtract from the template. The variance plane
743 of the science image is modified in place.
744 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
745 Exposure catalog with external calibrations to be applied. Catalog
746 uses the detector id for the catalog id, sorted on id for fast
747 lookup.
748 """
749 self._validateExposures(template, science)
750 if visitSummary is not None:
751 self._applyExternalCalibrations(science, visitSummary=visitSummary)
752 checkTemplateIsSufficient(template, self.log,
753 requiredTemplateFraction=self.config.requiredTemplateFraction)
755 if self.config.doScaleVariance:
756 # Scale the variance of the template and science images before
757 # convolution, subtraction, or decorrelation so that they have the
758 # correct ratio.
759 templateVarFactor = self.scaleVariance.run(template.maskedImage)
760 sciVarFactor = self.scaleVariance.run(science.maskedImage)
761 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
762 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
763 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
764 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor)
765 self._clearMask(template)
767 def _clearMask(self, template):
768 """Clear the mask plane of the template.
770 Parameters
771 ----------
772 template : `lsst.afw.image.ExposureF`
773 Template exposure, warped to match the science exposure.
774 The mask plane will be modified in place.
775 """
776 mask = template.mask
777 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys()
778 if maskplane not in self.config.preserveTemplateMask]
780 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes)
781 mask &= ~bitMaskToClear
784class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections,
785 SubtractScoreOutputConnections):
786 pass
789class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
790 pipelineConnections=AlardLuptonPreconvolveSubtractConnections):
791 pass
794class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask):
795 """Subtract a template from a science image, convolving the science image
796 before computing the kernel, and also convolving the template before
797 subtraction.
798 """
799 ConfigClass = AlardLuptonPreconvolveSubtractConfig
800 _DefaultName = "alardLuptonPreconvolveSubtract"
802 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
803 """Preconvolve the science image with its own PSF,
804 convolve the template image with a PSF-matching kernel and subtract
805 from the preconvolved science image.
807 Parameters
808 ----------
809 template : `lsst.afw.image.ExposureF`
810 The template image, which has previously been warped to the science
811 image. The template bbox will be padded by a few pixels compared to
812 the science bbox.
813 science : `lsst.afw.image.ExposureF`
814 The science exposure.
815 sources : `lsst.afw.table.SourceCatalog`
816 Identified sources on the science exposure. This catalog is used to
817 select sources in order to perform the AL PSF matching on stamp
818 images around them.
819 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
820 Exposure catalog with finalized psf models and aperture correction
821 maps to be applied. Catalog uses the detector id for the catalog
822 id, sorted on id for fast lookup. Deprecated in favor of
823 ``visitSummary``, and will be removed after v26.
824 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
825 Exposure catalog with complete external calibrations. Catalog uses
826 the detector id for the catalog id, sorted on id for fast lookup.
827 Ignored (for temporary backwards compatibility) if
828 ``finalizedPsfApCorrCatalog`` is provided.
830 Returns
831 -------
832 results : `lsst.pipe.base.Struct`
833 ``scoreExposure`` : `lsst.afw.image.ExposureF`
834 Result of subtracting the convolved template and science
835 images. Attached PSF is that of the original science image.
836 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
837 Warped and PSF-matched template exposure. Attached PSF is that
838 of the original science image.
839 ``matchedScience`` : `lsst.afw.image.ExposureF`
840 The science exposure after convolving with its own PSF.
841 Attached PSF is that of the original science image.
842 ``backgroundModel`` : `lsst.afw.math.Function2D`
843 Background model that was fit while solving for the
844 PSF-matching kernel
845 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
846 Final kernel used to PSF-match the template to the science
847 image.
848 """
849 if finalizedPsfApCorrCatalog is not None:
850 warnings.warn(
851 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
852 "argument, and will be removed after v26.",
853 FutureWarning,
854 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
855 )
856 visitSummary = finalizedPsfApCorrCatalog
858 self._prepareInputs(template, science, visitSummary=visitSummary)
860 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation
861 scienceKernel = science.psf.getKernel()
862 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl,
863 interpolateBadMaskPlanes=True)
864 selectSources = self._sourceSelector(sources, matchedScience.mask)
865 self.metadata.add("convolvedExposure", "Preconvolution")
867 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel)
869 return subtractResults
871 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel):
872 """Convolve the science image with its own PSF, then convolve the
873 template with a matching kernel and subtract to form the Score
874 exposure.
876 Parameters
877 ----------
878 template : `lsst.afw.image.ExposureF`
879 Template exposure, warped to match the science exposure.
880 science : `lsst.afw.image.ExposureF`
881 Science exposure to subtract from the template.
882 matchedScience : `lsst.afw.image.ExposureF`
883 The science exposure, convolved with the reflection of its own PSF.
884 selectSources : `lsst.afw.table.SourceCatalog`
885 Identified sources on the science exposure. This catalog is used to
886 select sources in order to perform the AL PSF matching on stamp
887 images around them.
888 preConvKernel : `lsst.afw.math.Kernel`
889 The reflection of the kernel that was used to preconvolve the
890 `science` exposure. Must be normalized to sum to 1.
892 Returns
893 -------
894 results : `lsst.pipe.base.Struct`
896 ``scoreExposure`` : `lsst.afw.image.ExposureF`
897 Result of subtracting the convolved template and science
898 images. Attached PSF is that of the original science image.
899 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
900 Warped and PSF-matched template exposure. Attached PSF is that
901 of the original science image.
902 ``matchedScience`` : `lsst.afw.image.ExposureF`
903 The science exposure after convolving with its own PSF.
904 Attached PSF is that of the original science image.
905 ``backgroundModel`` : `lsst.afw.math.Function2D`
906 Background model that was fit while solving for the
907 PSF-matching kernel
908 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
909 Final kernel used to PSF-match the template to the science
910 image.
911 """
912 bbox = science.getBBox()
913 innerBBox = preConvKernel.shrinkBBox(bbox)
915 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox],
916 candidateList=selectSources,
917 preconvolved=True)
918 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources,
919 preconvolved=True)
921 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
922 self.convolutionControl,
923 bbox=bbox,
924 psf=science.psf,
925 interpolateBadMaskPlanes=True,
926 photoCalib=science.photoCalib)
927 score = _subtractImages(matchedScience, matchedTemplate,
928 backgroundModel=(kernelResult.backgroundModel
929 if self.config.doSubtractBackground else None))
930 correctedScore = self.finalize(template[bbox], science, score,
931 kernelResult.psfMatchingKernel,
932 templateMatched=True, preConvMode=True,
933 preConvKernel=preConvKernel)
935 return lsst.pipe.base.Struct(scoreExposure=correctedScore,
936 matchedTemplate=matchedTemplate,
937 matchedScience=matchedScience,
938 backgroundModel=kernelResult.backgroundModel,
939 psfMatchingKernel=kernelResult.psfMatchingKernel)
942def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.):
943 """Raise NoWorkFound if template coverage < requiredTemplateFraction
945 Parameters
946 ----------
947 templateExposure : `lsst.afw.image.ExposureF`
948 The template exposure to check
949 logger : `lsst.log.Log`
950 Logger for printing output.
951 requiredTemplateFraction : `float`, optional
952 Fraction of pixels of the science image required to have coverage
953 in the template.
955 Raises
956 ------
957 lsst.pipe.base.NoWorkFound
958 Raised if fraction of good pixels, defined as not having NO_DATA
959 set, is less then the configured requiredTemplateFraction
960 """
961 # Count the number of pixels with the NO_DATA mask bit set
962 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
963 pixNoData = np.count_nonzero(templateExposure.mask.array
964 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
965 pixGood = templateExposure.getBBox().getArea() - pixNoData
966 logger.info("template has %d good pixels (%.1f%%)", pixGood,
967 100*pixGood/templateExposure.getBBox().getArea())
969 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
970 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
971 "To force subtraction, set config requiredTemplateFraction=0." % (
972 100*pixGood/templateExposure.getBBox().getArea(),
973 100*requiredTemplateFraction))
974 raise lsst.pipe.base.NoWorkFound(message)
977def _subtractImages(science, template, backgroundModel=None):
978 """Subtract template from science, propagating relevant metadata.
980 Parameters
981 ----------
982 science : `lsst.afw.Exposure`
983 The input science image.
984 template : `lsst.afw.Exposure`
985 The template to subtract from the science image.
986 backgroundModel : `lsst.afw.MaskedImage`, optional
987 Differential background model
989 Returns
990 -------
991 difference : `lsst.afw.Exposure`
992 The subtracted image.
993 """
994 difference = science.clone()
995 if backgroundModel is not None:
996 difference.maskedImage -= backgroundModel
997 difference.maskedImage -= template.maskedImage
998 return difference
1001def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
1002 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
1004 Parameters
1005 ----------
1006 exp1 : `~lsst.afw.image.Exposure`
1007 Exposure with the reference point spread function (PSF) to evaluate.
1008 exp2 : `~lsst.afw.image.Exposure`
1009 Exposure with a candidate point spread function (PSF) to evaluate.
1010 fwhmExposureBuffer : `float`
1011 Fractional buffer margin to be left out of all sides of the image
1012 during the construction of the grid to compute mean PSF FWHM in an
1013 exposure, if the PSF is not available at its average position.
1014 fwhmExposureGrid : `int`
1015 Grid size to compute the mean FWHM in an exposure, if the PSF is not
1016 available at its average position.
1017 Returns
1018 -------
1019 result : `bool`
1020 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in
1021 either dimension.
1022 """
1023 try:
1024 shape1 = getPsfFwhm(exp1.psf, average=False)
1025 shape2 = getPsfFwhm(exp2.psf, average=False)
1026 except InvalidParameterError:
1027 shape1 = evaluateMeanPsfFwhm(exp1,
1028 fwhmExposureBuffer=fwhmExposureBuffer,
1029 fwhmExposureGrid=fwhmExposureGrid
1030 )
1031 shape2 = evaluateMeanPsfFwhm(exp2,
1032 fwhmExposureBuffer=fwhmExposureBuffer,
1033 fwhmExposureGrid=fwhmExposureGrid
1034 )
1035 return shape1 <= shape2
1037 # Results from getPsfFwhm is a tuple of two values, one for each dimension.
1038 xTest = shape1[0] <= shape2[0]
1039 yTest = shape1[1] <= shape2[1]
1040 return xTest | yTest
1043def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None):
1044 """Replace masked image pixels with interpolated values.
1046 Parameters
1047 ----------
1048 maskedImage : `lsst.afw.image.MaskedImage`
1049 Image on which to perform interpolation.
1050 badMaskPlanes : `list` of `str`
1051 List of mask planes to interpolate over.
1052 fallbackValue : `float`, optional
1053 Value to set when interpolation fails.
1055 Returns
1056 -------
1057 result: `float`
1058 The number of masked pixels that were replaced.
1059 """
1060 image = maskedImage.image.array
1061 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(badMaskPlanes)) > 0
1062 image[badPixels] = np.nan
1063 if fallbackValue is None:
1064 fallbackValue = np.nanmedian(image)
1065 # For this initial implementation, skip the interpolation and just fill with
1066 # the median value.
1067 image[badPixels] = fallbackValue
1068 return np.sum(badPixels)