Coverage for python/lsst/ip/diffim/subtractImages.py: 23%
316 statements
« prev ^ index » next coverage.py v7.3.3, created at 2023-12-16 13:38 +0000
« prev ^ index » next coverage.py v7.3.3, created at 2023-12-16 13:38 +0000
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import warnings
24import numpy as np
26import lsst.afw.image
27import lsst.afw.math
28import lsst.geom
29from lsst.utils.introspection import find_outside_stacklevel
30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm
31from lsst.meas.algorithms import ScaleVarianceTask
32import lsst.pex.config
33import lsst.pipe.base
34import lsst.pex.exceptions
35from lsst.pipe.base import connectionTypes
36from . import MakeKernelTask, DecorrelateALKernelTask
37from lsst.utils.timer import timeMethod
39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask",
40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"]
42_dimensions = ("instrument", "visit", "detector")
43_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
47 dimensions=_dimensions,
48 defaultTemplates=_defaultTemplates):
49 template = connectionTypes.Input(
50 doc="Input warped template to subtract.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}{coaddName}Diff_templateExp"
54 )
55 science = connectionTypes.Input(
56 doc="Input science exposure to subtract from.",
57 dimensions=("instrument", "visit", "detector"),
58 storageClass="ExposureF",
59 name="{fakesType}calexp"
60 )
61 sources = connectionTypes.Input(
62 doc="Sources measured on the science exposure; "
63 "used to select sources for making the matching kernel.",
64 dimensions=("instrument", "visit", "detector"),
65 storageClass="SourceCatalog",
66 name="{fakesType}src"
67 )
68 finalizedPsfApCorrCatalog = connectionTypes.Input(
69 doc=("Per-visit finalized psf models and aperture correction maps. "
70 "These catalogs use the detector id for the catalog id, "
71 "sorted on id for fast lookup."),
72 dimensions=("instrument", "visit"),
73 storageClass="ExposureCatalog",
74 name="finalVisitSummary",
75 # TODO: remove on DM-39854.
76 deprecated=(
77 "Deprecated in favor of visitSummary. Will be removed after v26."
78 )
79 )
80 visitSummary = connectionTypes.Input(
81 doc=("Per-visit catalog with final calibration objects. "
82 "These catalogs use the detector id for the catalog id, "
83 "sorted on id for fast lookup."),
84 dimensions=("instrument", "visit"),
85 storageClass="ExposureCatalog",
86 name="finalVisitSummary",
87 )
89 def __init__(self, *, config=None):
90 super().__init__(config=config)
91 if not config.doApplyFinalizedPsf:
92 self.inputs.remove("finalizedPsfApCorrCatalog")
93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf:
94 del self.visitSummary
97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
98 dimensions=_dimensions,
99 defaultTemplates=_defaultTemplates):
100 difference = connectionTypes.Output(
101 doc="Result of subtracting convolved template from science image.",
102 dimensions=("instrument", "visit", "detector"),
103 storageClass="ExposureF",
104 name="{fakesType}{coaddName}Diff_differenceTempExp",
105 )
106 matchedTemplate = connectionTypes.Output(
107 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
108 dimensions=("instrument", "visit", "detector"),
109 storageClass="ExposureF",
110 name="{fakesType}{coaddName}Diff_matchedExp",
111 )
114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections,
115 dimensions=_dimensions,
116 defaultTemplates=_defaultTemplates):
117 scoreExposure = connectionTypes.Output(
118 doc="The maximum likelihood image, used for the detection of diaSources.",
119 dimensions=("instrument", "visit", "detector"),
120 storageClass="ExposureF",
121 name="{fakesType}{coaddName}Diff_scoreExp",
122 )
125class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections):
126 pass
129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
130 makeKernel = lsst.pex.config.ConfigurableField(
131 target=MakeKernelTask,
132 doc="Task to construct a matching kernel for convolution.",
133 )
134 doDecorrelation = lsst.pex.config.Field(
135 dtype=bool,
136 default=True,
137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
138 "kernel convolution? If True, also update the diffim PSF."
139 )
140 decorrelate = lsst.pex.config.ConfigurableField(
141 target=DecorrelateALKernelTask,
142 doc="Task to decorrelate the image difference.",
143 )
144 requiredTemplateFraction = lsst.pex.config.Field(
145 dtype=float,
146 default=0.1,
147 doc="Raise NoWorkFound and do not attempt image subtraction if template covers less than this "
148 " fraction of pixels. Setting to 0 will always attempt image subtraction."
149 )
150 minTemplateFractionForExpectedSuccess = lsst.pex.config.Field(
151 dtype=float,
152 default=0.2,
153 doc="Raise NoWorkFound if PSF-matching fails and template covers less than this fraction of pixels."
154 " If the fraction of pixels covered by the template is less than this value (and greater than"
155 " requiredTemplateFraction) this task is attempted but failure is anticipated and tolerated."
156 )
157 doScaleVariance = lsst.pex.config.Field(
158 dtype=bool,
159 default=True,
160 doc="Scale variance of the image difference?"
161 )
162 scaleVariance = lsst.pex.config.ConfigurableField(
163 target=ScaleVarianceTask,
164 doc="Subtask to rescale the variance of the template to the statistically expected level."
165 )
166 doSubtractBackground = lsst.pex.config.Field(
167 doc="Subtract the background fit when solving the kernel?",
168 dtype=bool,
169 default=True,
170 )
171 doApplyFinalizedPsf = lsst.pex.config.Field(
172 doc="Replace science Exposure's psf and aperture correction map"
173 " with those in finalizedPsfApCorrCatalog.",
174 dtype=bool,
175 default=False,
176 # TODO: remove on DM-39854.
177 deprecated=(
178 "Deprecated in favor of doApplyExternalCalibrations. "
179 "Will be removed after v26."
180 )
181 )
182 doApplyExternalCalibrations = lsst.pex.config.Field(
183 doc=(
184 "Replace science Exposure's calibration objects with those"
185 " in visitSummary. Ignored if `doApplyFinalizedPsf is True."
186 ),
187 dtype=bool,
188 default=False,
189 )
190 detectionThreshold = lsst.pex.config.Field(
191 dtype=float,
192 default=10,
193 doc="Minimum signal to noise ratio of detected sources "
194 "to use for calculating the PSF matching kernel."
195 )
196 badSourceFlags = lsst.pex.config.ListField(
197 dtype=str,
198 doc="Flags that, if set, the associated source should not "
199 "be used to determine the PSF matching kernel.",
200 default=("sky_source", "slot_Centroid_flag",
201 "slot_ApFlux_flag", "slot_PsfFlux_flag", ),
202 )
203 excludeMaskPlanes = lsst.pex.config.ListField(
204 dtype=str,
205 default=("NO_DATA", "BAD", "SAT", "EDGE", "FAKE"),
206 doc="Mask planes to exclude when selecting sources for PSF matching."
207 )
208 badMaskPlanes = lsst.pex.config.ListField(
209 dtype=str,
210 default=("NO_DATA", "BAD", "SAT", "EDGE"),
211 doc="Mask planes to interpolate over."
212 )
213 preserveTemplateMask = lsst.pex.config.ListField(
214 dtype=str,
215 default=("NO_DATA", "BAD", "SAT", "FAKE", "INJECTED", "INJECTED_CORE"),
216 doc="Mask planes from the template to propagate to the image difference."
217 )
218 allowKernelSourceDetection = lsst.pex.config.Field(
219 dtype=bool,
220 default=False,
221 doc="Re-run source detection for kernel candidates if an error is"
222 " encountered while calculating the matching kernel."
223 )
225 def setDefaults(self):
226 self.makeKernel.kernel.name = "AL"
227 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground
228 self.makeKernel.kernel.active.spatialKernelOrder = 1
229 self.makeKernel.kernel.active.spatialBgOrder = 2
232class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
233 pipelineConnections=AlardLuptonSubtractConnections):
234 mode = lsst.pex.config.ChoiceField(
235 dtype=str,
236 default="convolveTemplate",
237 allowed={"auto": "Choose which image to convolve at runtime.",
238 "convolveScience": "Only convolve the science image.",
239 "convolveTemplate": "Only convolve the template image."},
240 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
241 )
244class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
245 """Compute the image difference of a science and template image using
246 the Alard & Lupton (1998) algorithm.
247 """
248 ConfigClass = AlardLuptonSubtractConfig
249 _DefaultName = "alardLuptonSubtract"
251 def __init__(self, **kwargs):
252 super().__init__(**kwargs)
253 self.makeSubtask("decorrelate")
254 self.makeSubtask("makeKernel")
255 if self.config.doScaleVariance:
256 self.makeSubtask("scaleVariance")
258 self.convolutionControl = lsst.afw.math.ConvolutionControl()
259 # Normalization is an extra, unnecessary, calculation and will result
260 # in mis-subtraction of the images if there are calibration errors.
261 self.convolutionControl.setDoNormalize(False)
262 self.convolutionControl.setDoCopyEdge(True)
264 def _applyExternalCalibrations(self, exposure, visitSummary):
265 """Replace calibrations (psf, and ApCorrMap) on this exposure with
266 external ones.".
268 Parameters
269 ----------
270 exposure : `lsst.afw.image.exposure.Exposure`
271 Input exposure to adjust calibrations.
272 visitSummary : `lsst.afw.table.ExposureCatalog`
273 Exposure catalog with external calibrations to be applied. Catalog
274 uses the detector id for the catalog id, sorted on id for fast
275 lookup.
277 Returns
278 -------
279 exposure : `lsst.afw.image.exposure.Exposure`
280 Exposure with adjusted calibrations.
281 """
282 detectorId = exposure.info.getDetector().getId()
284 row = visitSummary.find(detectorId)
285 if row is None:
286 self.log.warning("Detector id %s not found in external calibrations catalog; "
287 "Using original calibrations.", detectorId)
288 else:
289 psf = row.getPsf()
290 apCorrMap = row.getApCorrMap()
291 if psf is None:
292 self.log.warning("Detector id %s has None for psf in "
293 "external calibrations catalog; Using original psf and aperture correction.",
294 detectorId)
295 elif apCorrMap is None:
296 self.log.warning("Detector id %s has None for apCorrMap in "
297 "external calibrations catalog; Using original psf and aperture correction.",
298 detectorId)
299 else:
300 exposure.setPsf(psf)
301 exposure.info.setApCorrMap(apCorrMap)
303 return exposure
305 @timeMethod
306 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None,
307 visitSummary=None):
308 """PSF match, subtract, and decorrelate two images.
310 Parameters
311 ----------
312 template : `lsst.afw.image.ExposureF`
313 Template exposure, warped to match the science exposure.
314 science : `lsst.afw.image.ExposureF`
315 Science exposure to subtract from the template.
316 sources : `lsst.afw.table.SourceCatalog`
317 Identified sources on the science exposure. This catalog is used to
318 select sources in order to perform the AL PSF matching on stamp
319 images around them.
320 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
321 Exposure catalog with finalized psf models and aperture correction
322 maps to be applied. Catalog uses the detector id for the catalog
323 id, sorted on id for fast lookup. Deprecated in favor of
324 ``visitSummary``, and will be removed after v26.
325 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
326 Exposure catalog with external calibrations to be applied. Catalog
327 uses the detector id for the catalog id, sorted on id for fast
328 lookup. Ignored (for temporary backwards compatibility) if
329 ``finalizedPsfApCorrCatalog`` is provided.
331 Returns
332 -------
333 results : `lsst.pipe.base.Struct`
334 ``difference`` : `lsst.afw.image.ExposureF`
335 Result of subtracting template and science.
336 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
337 Warped and PSF-matched template exposure.
338 ``backgroundModel`` : `lsst.afw.math.Function2D`
339 Background model that was fit while solving for the
340 PSF-matching kernel
341 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
342 Kernel used to PSF-match the convolved image.
344 Raises
345 ------
346 RuntimeError
347 If an unsupported convolution mode is supplied.
348 RuntimeError
349 If there are too few sources to calculate the PSF matching kernel.
350 lsst.pipe.base.NoWorkFound
351 Raised if fraction of good pixels, defined as not having NO_DATA
352 set, is less then the configured requiredTemplateFraction
353 """
355 if finalizedPsfApCorrCatalog is not None:
356 warnings.warn(
357 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
358 "argument, and will be removed after v26.",
359 FutureWarning,
360 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
361 )
362 visitSummary = finalizedPsfApCorrCatalog
364 self._prepareInputs(template, science, visitSummary=visitSummary)
366 # In the event that getPsfFwhm fails, evaluate the PSF on a grid.
367 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
368 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
370 # Calling getPsfFwhm on template.psf fails on some rare occasions when
371 # the template has no input exposures at the average position of the
372 # stars. So we try getPsfFwhm first on template, and if that fails we
373 # evaluate the PSF on a grid specified by fwhmExposure* fields.
374 # To keep consistent definitions for PSF size on the template and
375 # science images, we use the same method for both.
376 try:
377 templatePsfSize = getPsfFwhm(template.psf)
378 sciencePsfSize = getPsfFwhm(science.psf)
379 except lsst.pex.exceptions.InvalidParameterError:
380 self.log.info("Unable to evaluate PSF at the average position. "
381 "Evaluting PSF on a grid of points."
382 )
383 templatePsfSize = evaluateMeanPsfFwhm(template,
384 fwhmExposureBuffer=fwhmExposureBuffer,
385 fwhmExposureGrid=fwhmExposureGrid
386 )
387 sciencePsfSize = evaluateMeanPsfFwhm(science,
388 fwhmExposureBuffer=fwhmExposureBuffer,
389 fwhmExposureGrid=fwhmExposureGrid
390 )
391 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize)
392 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize)
393 self.metadata.add("sciencePsfSize", sciencePsfSize)
394 self.metadata.add("templatePsfSize", templatePsfSize)
396 if self.config.mode == "auto":
397 convolveTemplate = _shapeTest(template,
398 science,
399 fwhmExposureBuffer=fwhmExposureBuffer,
400 fwhmExposureGrid=fwhmExposureGrid)
401 if convolveTemplate:
402 if sciencePsfSize < templatePsfSize:
403 self.log.info("Average template PSF size is greater, "
404 "but science PSF greater in one dimension: convolving template image.")
405 else:
406 self.log.info("Science PSF size is greater: convolving template image.")
407 else:
408 self.log.info("Template PSF size is greater: convolving science image.")
409 elif self.config.mode == "convolveTemplate":
410 self.log.info("`convolveTemplate` is set: convolving template image.")
411 convolveTemplate = True
412 elif self.config.mode == "convolveScience":
413 self.log.info("`convolveScience` is set: convolving science image.")
414 convolveTemplate = False
415 else:
416 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
418 try:
419 selectSources = self._sourceSelector(sources, science.mask)
420 if convolveTemplate:
421 self.metadata.add("convolvedExposure", "Template")
422 subtractResults = self.runConvolveTemplate(template, science, selectSources)
423 else:
424 self.metadata.add("convolvedExposure", "Science")
425 subtractResults = self.runConvolveScience(template, science, selectSources)
427 except (RuntimeError, lsst.pex.exceptions.Exception) as e:
428 self.log.warning("Failed to match template. Checking coverage")
429 # Raise NoWorkFound if template fraction is insufficient
430 checkTemplateIsSufficient(template[science.getBBox()], self.log,
431 self.config.minTemplateFractionForExpectedSuccess,
432 exceptionMessage="Template coverage lower than expected to succeed."
433 f" Failure is tolerable: {e}")
434 # checkTemplateIsSufficient did not raise NoWorkFound, so raise original exception
435 raise e
437 return subtractResults
439 def runConvolveTemplate(self, template, science, selectSources):
440 """Convolve the template image with a PSF-matching kernel and subtract
441 from the science image.
443 Parameters
444 ----------
445 template : `lsst.afw.image.ExposureF`
446 Template exposure, warped to match the science exposure.
447 science : `lsst.afw.image.ExposureF`
448 Science exposure to subtract from the template.
449 selectSources : `lsst.afw.table.SourceCatalog`
450 Identified sources on the science exposure. This catalog is used to
451 select sources in order to perform the AL PSF matching on stamp
452 images around them.
454 Returns
455 -------
456 results : `lsst.pipe.base.Struct`
458 ``difference`` : `lsst.afw.image.ExposureF`
459 Result of subtracting template and science.
460 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
461 Warped and PSF-matched template exposure.
462 ``backgroundModel`` : `lsst.afw.math.Function2D`
463 Background model that was fit while solving for the PSF-matching kernel
464 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
465 Kernel used to PSF-match the template to the science image.
466 """
467 try:
468 kernelSources = self.makeKernel.selectKernelSources(template, science,
469 candidateList=selectSources,
470 preconvolved=False)
471 kernelResult = self.makeKernel.run(template, science, kernelSources,
472 preconvolved=False)
473 except Exception as e:
474 if self.config.allowKernelSourceDetection:
475 self.log.warning("Error encountered trying to construct the matching kernel"
476 f" Running source detection and retrying. {e}")
477 kernelSources = self.makeKernel.selectKernelSources(template, science,
478 candidateList=None,
479 preconvolved=False)
480 kernelResult = self.makeKernel.run(template, science, kernelSources,
481 preconvolved=False)
482 else:
483 raise e
485 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
486 self.convolutionControl,
487 bbox=science.getBBox(),
488 psf=science.psf,
489 photoCalib=science.photoCalib)
491 difference = _subtractImages(science, matchedTemplate,
492 backgroundModel=(kernelResult.backgroundModel
493 if self.config.doSubtractBackground else None))
494 correctedExposure = self.finalize(template, science, difference,
495 kernelResult.psfMatchingKernel,
496 templateMatched=True)
498 return lsst.pipe.base.Struct(difference=correctedExposure,
499 matchedTemplate=matchedTemplate,
500 matchedScience=science,
501 backgroundModel=kernelResult.backgroundModel,
502 psfMatchingKernel=kernelResult.psfMatchingKernel)
504 def runConvolveScience(self, template, science, selectSources):
505 """Convolve the science image with a PSF-matching kernel and subtract
506 the template image.
508 Parameters
509 ----------
510 template : `lsst.afw.image.ExposureF`
511 Template exposure, warped to match the science exposure.
512 science : `lsst.afw.image.ExposureF`
513 Science exposure to subtract from the template.
514 selectSources : `lsst.afw.table.SourceCatalog`
515 Identified sources on the science exposure. This catalog is used to
516 select sources in order to perform the AL PSF matching on stamp
517 images around them.
519 Returns
520 -------
521 results : `lsst.pipe.base.Struct`
523 ``difference`` : `lsst.afw.image.ExposureF`
524 Result of subtracting template and science.
525 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
526 Warped template exposure. Note that in this case, the template
527 is not PSF-matched to the science image.
528 ``backgroundModel`` : `lsst.afw.math.Function2D`
529 Background model that was fit while solving for the PSF-matching kernel
530 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
531 Kernel used to PSF-match the science image to the template.
532 """
533 bbox = science.getBBox()
534 kernelSources = self.makeKernel.selectKernelSources(science, template,
535 candidateList=selectSources,
536 preconvolved=False)
537 kernelResult = self.makeKernel.run(science, template, kernelSources,
538 preconvolved=False)
539 modelParams = kernelResult.backgroundModel.getParameters()
540 # We must invert the background model if the matching kernel is solved for the science image.
541 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
543 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
544 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False)
546 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel,
547 self.convolutionControl,
548 psf=template.psf)
550 # Place back on native photometric scale
551 matchedScience.maskedImage /= norm
552 matchedTemplate = template.clone()[bbox]
553 matchedTemplate.maskedImage /= norm
554 matchedTemplate.setPhotoCalib(science.photoCalib)
556 difference = _subtractImages(matchedScience, matchedTemplate,
557 backgroundModel=(kernelResult.backgroundModel
558 if self.config.doSubtractBackground else None))
560 correctedExposure = self.finalize(template, science, difference,
561 kernelResult.psfMatchingKernel,
562 templateMatched=False)
564 return lsst.pipe.base.Struct(difference=correctedExposure,
565 matchedTemplate=matchedTemplate,
566 matchedScience=matchedScience,
567 backgroundModel=kernelResult.backgroundModel,
568 psfMatchingKernel=kernelResult.psfMatchingKernel,)
570 def finalize(self, template, science, difference, kernel,
571 templateMatched=True,
572 preConvMode=False,
573 preConvKernel=None,
574 spatiallyVarying=False):
575 """Decorrelate the difference image to undo the noise correlations
576 caused by convolution.
578 Parameters
579 ----------
580 template : `lsst.afw.image.ExposureF`
581 Template exposure, warped to match the science exposure.
582 science : `lsst.afw.image.ExposureF`
583 Science exposure to subtract from the template.
584 difference : `lsst.afw.image.ExposureF`
585 Result of subtracting template and science.
586 kernel : `lsst.afw.math.Kernel`
587 An (optionally spatially-varying) PSF matching kernel
588 templateMatched : `bool`, optional
589 Was the template PSF-matched to the science image?
590 preConvMode : `bool`, optional
591 Was the science image preconvolved with its own PSF
592 before PSF matching the template?
593 preConvKernel : `lsst.afw.detection.Psf`, optional
594 If not `None`, then the science image was pre-convolved with
595 (the reflection of) this kernel. Must be normalized to sum to 1.
596 spatiallyVarying : `bool`, optional
597 Compute the decorrelation kernel spatially varying across the image?
599 Returns
600 -------
601 correctedExposure : `lsst.afw.image.ExposureF`
602 The decorrelated image difference.
603 """
604 # Erase existing detection mask planes.
605 # We don't want the detection mask from the science image
607 self.updateMasks(template, science, difference)
609 if self.config.doDecorrelation:
610 self.log.info("Decorrelating image difference.")
611 # We have cleared the template mask plane, so copy the mask plane of
612 # the image difference so that we can calculate correct statistics
613 # during decorrelation
614 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
615 templateMatched=templateMatched,
616 preConvMode=preConvMode,
617 preConvKernel=preConvKernel,
618 spatiallyVarying=spatiallyVarying).correctedExposure
619 else:
620 self.log.info("NOT decorrelating image difference.")
621 correctedExposure = difference
622 return correctedExposure
624 def updateMasks(self, template, science, difference):
625 """Update the mask planes on images for finalizing."""
627 bbox = science.getBBox()
628 mask = difference.mask
629 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
631 if "FAKE" in science.mask.getMaskPlaneDict().keys():
632 # propagate the mask plane related to Fake source injection
633 # NOTE: the fake source injection sets FAKE plane, but it should be INJECTED
634 # NOTE: This can be removed in DM-40796
636 self.log.info("Adding injected mask planes")
637 mask.addMaskPlane("INJECTED")
638 diffInjectedBitMask = mask.getPlaneBitMask("INJECTED")
640 mask.addMaskPlane("INJECTED_TEMPLATE")
641 diffInjTmpltBitMask = mask.getPlaneBitMask("INJECTED_TEMPLATE")
643 scienceFakeBitMask = science.mask.getPlaneBitMask('FAKE')
644 tmpltFakeBitMask = template[bbox].mask.getPlaneBitMask('FAKE')
646 injScienceMaskArray = ((science.mask.array & scienceFakeBitMask) > 0) * diffInjectedBitMask
647 injTemplateMaskArray = ((template[bbox].mask.array & tmpltFakeBitMask) > 0) * diffInjTmpltBitMask
649 mask.array |= injScienceMaskArray
650 mask.array |= injTemplateMaskArray
652 template[bbox].mask.array[...] = difference.mask.array[...]
654 @staticmethod
655 def _validateExposures(template, science):
656 """Check that the WCS of the two Exposures match, and the template bbox
657 contains the science bbox.
659 Parameters
660 ----------
661 template : `lsst.afw.image.ExposureF`
662 Template exposure, warped to match the science exposure.
663 science : `lsst.afw.image.ExposureF`
664 Science exposure to subtract from the template.
666 Raises
667 ------
668 AssertionError
669 Raised if the WCS of the template is not equal to the science WCS,
670 or if the science image is not fully contained in the template
671 bounding box.
672 """
673 assert template.wcs == science.wcs,\
674 "Template and science exposure WCS are not identical."
675 templateBBox = template.getBBox()
676 scienceBBox = science.getBBox()
678 assert templateBBox.contains(scienceBBox),\
679 "Template bbox does not contain all of the science image."
681 def _convolveExposure(self, exposure, kernel, convolutionControl,
682 bbox=None,
683 psf=None,
684 photoCalib=None,
685 interpolateBadMaskPlanes=False,
686 ):
687 """Convolve an exposure with the given kernel.
689 Parameters
690 ----------
691 exposure : `lsst.afw.Exposure`
692 exposure to convolve.
693 kernel : `lsst.afw.math.LinearCombinationKernel`
694 PSF matching kernel computed in the ``makeKernel`` subtask.
695 convolutionControl : `lsst.afw.math.ConvolutionControl`
696 Configuration for convolve algorithm.
697 bbox : `lsst.geom.Box2I`, optional
698 Bounding box to trim the convolved exposure to.
699 psf : `lsst.afw.detection.Psf`, optional
700 Point spread function (PSF) to set for the convolved exposure.
701 photoCalib : `lsst.afw.image.PhotoCalib`, optional
702 Photometric calibration of the convolved exposure.
704 Returns
705 -------
706 convolvedExp : `lsst.afw.Exposure`
707 The convolved image.
708 """
709 convolvedExposure = exposure.clone()
710 if psf is not None:
711 convolvedExposure.setPsf(psf)
712 if photoCalib is not None:
713 convolvedExposure.setPhotoCalib(photoCalib)
714 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None:
715 nInterp = _interpolateImage(convolvedExposure.maskedImage,
716 self.config.badMaskPlanes)
717 self.metadata.add("nInterpolated", nInterp)
718 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox())
719 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl)
720 convolvedExposure.setMaskedImage(convolvedImage)
721 if bbox is None:
722 return convolvedExposure
723 else:
724 return convolvedExposure[bbox]
726 def _sourceSelector(self, sources, mask):
727 """Select sources from a catalog that meet the selection criteria.
729 Parameters
730 ----------
731 sources : `lsst.afw.table.SourceCatalog`
732 Input source catalog to select sources from.
733 mask : `lsst.afw.image.Mask`
734 The image mask plane to use to reject sources
735 based on their location on the ccd.
737 Returns
738 -------
739 selectSources : `lsst.afw.table.SourceCatalog`
740 The input source catalog, with flagged and low signal-to-noise
741 sources removed.
743 Raises
744 ------
745 RuntimeError
746 If there are too few sources to compute the PSF matching kernel
747 remaining after source selection.
748 """
749 flags = np.ones(len(sources), dtype=bool)
750 for flag in self.config.badSourceFlags:
751 try:
752 flags *= ~sources[flag]
753 except Exception as e:
754 self.log.warning("Could not apply source flag: %s", e)
755 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
756 flags *= sToNFlag
757 flags *= self._checkMask(mask, sources, self.config.excludeMaskPlanes)
758 selectSources = sources[flags]
759 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog",
760 len(selectSources), len(sources), 100*len(selectSources)/len(sources))
761 if len(selectSources) < self.config.makeKernel.nStarPerCell:
762 self.log.error("Too few sources to calculate the PSF matching kernel: "
763 "%i selected but %i needed for the calculation.",
764 len(selectSources), self.config.makeKernel.nStarPerCell)
765 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.")
766 self.metadata.add("nPsfSources", len(selectSources))
768 return selectSources.copy(deep=True)
770 @staticmethod
771 def _checkMask(mask, sources, excludeMaskPlanes):
772 """Exclude sources that are located on masked pixels.
774 Parameters
775 ----------
776 mask : `lsst.afw.image.Mask`
777 The image mask plane to use to reject sources
778 based on the location of their centroid on the ccd.
779 sources : `lsst.afw.table.SourceCatalog`
780 The source catalog to evaluate.
781 excludeMaskPlanes : `list` of `str`
782 List of the names of the mask planes to exclude.
784 Returns
785 -------
786 flags : `numpy.ndarray` of `bool`
787 Array indicating whether each source in the catalog should be
788 kept (True) or rejected (False) based on the value of the
789 mask plane at its location.
790 """
791 setExcludeMaskPlanes = [
792 maskPlane for maskPlane in excludeMaskPlanes if maskPlane in mask.getMaskPlaneDict()
793 ]
795 excludePixelMask = mask.getPlaneBitMask(setExcludeMaskPlanes)
797 xv = np.rint(sources.getX() - mask.getX0())
798 yv = np.rint(sources.getY() - mask.getY0())
800 mv = mask.array[yv.astype(int), xv.astype(int)]
801 flags = np.bitwise_and(mv, excludePixelMask) == 0
802 return flags
804 def _prepareInputs(self, template, science, visitSummary=None):
805 """Perform preparatory calculations common to all Alard&Lupton Tasks.
807 Parameters
808 ----------
809 template : `lsst.afw.image.ExposureF`
810 Template exposure, warped to match the science exposure. The
811 variance plane of the template image is modified in place.
812 science : `lsst.afw.image.ExposureF`
813 Science exposure to subtract from the template. The variance plane
814 of the science image is modified in place.
815 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
816 Exposure catalog with external calibrations to be applied. Catalog
817 uses the detector id for the catalog id, sorted on id for fast
818 lookup.
819 """
820 self._validateExposures(template, science)
821 if visitSummary is not None:
822 self._applyExternalCalibrations(science, visitSummary=visitSummary)
823 checkTemplateIsSufficient(template[science.getBBox()], self.log,
824 requiredTemplateFraction=self.config.requiredTemplateFraction,
825 exceptionMessage="Not attempting subtraction. To force subtraction,"
826 " set config requiredTemplateFraction=0")
828 if self.config.doScaleVariance:
829 # Scale the variance of the template and science images before
830 # convolution, subtraction, or decorrelation so that they have the
831 # correct ratio.
832 templateVarFactor = self.scaleVariance.run(template.maskedImage)
833 sciVarFactor = self.scaleVariance.run(science.maskedImage)
834 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
835 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
836 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
837 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor)
838 self._clearMask(template)
840 def _clearMask(self, template):
841 """Clear the mask plane of the template.
843 Parameters
844 ----------
845 template : `lsst.afw.image.ExposureF`
846 Template exposure, warped to match the science exposure.
847 The mask plane will be modified in place.
848 """
849 mask = template.mask
850 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys()
851 if maskplane not in self.config.preserveTemplateMask]
853 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes)
854 mask &= ~bitMaskToClear
857class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections,
858 SubtractScoreOutputConnections):
859 pass
862class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
863 pipelineConnections=AlardLuptonPreconvolveSubtractConnections):
864 pass
867class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask):
868 """Subtract a template from a science image, convolving the science image
869 before computing the kernel, and also convolving the template before
870 subtraction.
871 """
872 ConfigClass = AlardLuptonPreconvolveSubtractConfig
873 _DefaultName = "alardLuptonPreconvolveSubtract"
875 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
876 """Preconvolve the science image with its own PSF,
877 convolve the template image with a PSF-matching kernel and subtract
878 from the preconvolved science image.
880 Parameters
881 ----------
882 template : `lsst.afw.image.ExposureF`
883 The template image, which has previously been warped to the science
884 image. The template bbox will be padded by a few pixels compared to
885 the science bbox.
886 science : `lsst.afw.image.ExposureF`
887 The science exposure.
888 sources : `lsst.afw.table.SourceCatalog`
889 Identified sources on the science exposure. This catalog is used to
890 select sources in order to perform the AL PSF matching on stamp
891 images around them.
892 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
893 Exposure catalog with finalized psf models and aperture correction
894 maps to be applied. Catalog uses the detector id for the catalog
895 id, sorted on id for fast lookup. Deprecated in favor of
896 ``visitSummary``, and will be removed after v26.
897 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
898 Exposure catalog with complete external calibrations. Catalog uses
899 the detector id for the catalog id, sorted on id for fast lookup.
900 Ignored (for temporary backwards compatibility) if
901 ``finalizedPsfApCorrCatalog`` is provided.
903 Returns
904 -------
905 results : `lsst.pipe.base.Struct`
906 ``scoreExposure`` : `lsst.afw.image.ExposureF`
907 Result of subtracting the convolved template and science
908 images. Attached PSF is that of the original science image.
909 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
910 Warped and PSF-matched template exposure. Attached PSF is that
911 of the original science image.
912 ``matchedScience`` : `lsst.afw.image.ExposureF`
913 The science exposure after convolving with its own PSF.
914 Attached PSF is that of the original science image.
915 ``backgroundModel`` : `lsst.afw.math.Function2D`
916 Background model that was fit while solving for the
917 PSF-matching kernel
918 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
919 Final kernel used to PSF-match the template to the science
920 image.
921 """
922 if finalizedPsfApCorrCatalog is not None:
923 warnings.warn(
924 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
925 "argument, and will be removed after v26.",
926 FutureWarning,
927 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
928 )
929 visitSummary = finalizedPsfApCorrCatalog
931 self._prepareInputs(template, science, visitSummary=visitSummary)
933 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation
934 scienceKernel = science.psf.getKernel()
935 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl,
936 interpolateBadMaskPlanes=True)
937 self.metadata.add("convolvedExposure", "Preconvolution")
938 try:
939 selectSources = self._sourceSelector(sources, matchedScience.mask)
940 subtractResults = self.runPreconvolve(template, science, matchedScience,
941 selectSources, scienceKernel)
943 except (RuntimeError, lsst.pex.exceptions.Exception) as e:
944 self.log.warning("Failed to match template. Checking coverage")
945 # Raise NoWorkFound if template fraction is insufficient
946 checkTemplateIsSufficient(template[science.getBBox()], self.log,
947 self.config.minTemplateFractionForExpectedSuccess,
948 exceptionMessage="Template coverage lower than expected to succeed."
949 f" Failure is tolerable: {e}")
950 # checkTemplateIsSufficient did not raise NoWorkFound, so raise original exception
951 raise e
953 return subtractResults
955 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel):
956 """Convolve the science image with its own PSF, then convolve the
957 template with a matching kernel and subtract to form the Score
958 exposure.
960 Parameters
961 ----------
962 template : `lsst.afw.image.ExposureF`
963 Template exposure, warped to match the science exposure.
964 science : `lsst.afw.image.ExposureF`
965 Science exposure to subtract from the template.
966 matchedScience : `lsst.afw.image.ExposureF`
967 The science exposure, convolved with the reflection of its own PSF.
968 selectSources : `lsst.afw.table.SourceCatalog`
969 Identified sources on the science exposure. This catalog is used to
970 select sources in order to perform the AL PSF matching on stamp
971 images around them.
972 preConvKernel : `lsst.afw.math.Kernel`
973 The reflection of the kernel that was used to preconvolve the
974 `science` exposure. Must be normalized to sum to 1.
976 Returns
977 -------
978 results : `lsst.pipe.base.Struct`
980 ``scoreExposure`` : `lsst.afw.image.ExposureF`
981 Result of subtracting the convolved template and science
982 images. Attached PSF is that of the original science image.
983 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
984 Warped and PSF-matched template exposure. Attached PSF is that
985 of the original science image.
986 ``matchedScience`` : `lsst.afw.image.ExposureF`
987 The science exposure after convolving with its own PSF.
988 Attached PSF is that of the original science image.
989 ``backgroundModel`` : `lsst.afw.math.Function2D`
990 Background model that was fit while solving for the
991 PSF-matching kernel
992 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
993 Final kernel used to PSF-match the template to the science
994 image.
995 """
996 bbox = science.getBBox()
997 innerBBox = preConvKernel.shrinkBBox(bbox)
999 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox],
1000 candidateList=selectSources,
1001 preconvolved=True)
1002 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources,
1003 preconvolved=True)
1005 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
1006 self.convolutionControl,
1007 bbox=bbox,
1008 psf=science.psf,
1009 interpolateBadMaskPlanes=True,
1010 photoCalib=science.photoCalib)
1011 score = _subtractImages(matchedScience, matchedTemplate,
1012 backgroundModel=(kernelResult.backgroundModel
1013 if self.config.doSubtractBackground else None))
1014 correctedScore = self.finalize(template[bbox], science, score,
1015 kernelResult.psfMatchingKernel,
1016 templateMatched=True, preConvMode=True,
1017 preConvKernel=preConvKernel)
1019 return lsst.pipe.base.Struct(scoreExposure=correctedScore,
1020 matchedTemplate=matchedTemplate,
1021 matchedScience=matchedScience,
1022 backgroundModel=kernelResult.backgroundModel,
1023 psfMatchingKernel=kernelResult.psfMatchingKernel)
1026def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.,
1027 exceptionMessage=""):
1028 """Raise NoWorkFound if template coverage < requiredTemplateFraction
1030 Parameters
1031 ----------
1032 templateExposure : `lsst.afw.image.ExposureF`
1033 The template exposure to check
1034 logger : `lsst.log.Log`
1035 Logger for printing output.
1036 requiredTemplateFraction : `float`, optional
1037 Fraction of pixels of the science image required to have coverage
1038 in the template.
1039 exceptionMessage : `str`, optional
1040 Message to include in the exception raised if the template coverage
1041 is insufficient.
1043 Raises
1044 ------
1045 lsst.pipe.base.NoWorkFound
1046 Raised if fraction of good pixels, defined as not having NO_DATA
1047 set, is less than the requiredTemplateFraction
1048 """
1049 # Count the number of pixels with the NO_DATA mask bit set
1050 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1051 pixNoData = np.count_nonzero(templateExposure.mask.array
1052 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1053 pixGood = templateExposure.getBBox().getArea() - pixNoData
1054 logger.info("template has %d good pixels (%.1f%%)", pixGood,
1055 100*pixGood/templateExposure.getBBox().getArea())
1057 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
1058 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%)" % (
1059 100*pixGood/templateExposure.getBBox().getArea(),
1060 100*requiredTemplateFraction))
1061 raise lsst.pipe.base.NoWorkFound(message + " " + exceptionMessage)
1064def _subtractImages(science, template, backgroundModel=None):
1065 """Subtract template from science, propagating relevant metadata.
1067 Parameters
1068 ----------
1069 science : `lsst.afw.Exposure`
1070 The input science image.
1071 template : `lsst.afw.Exposure`
1072 The template to subtract from the science image.
1073 backgroundModel : `lsst.afw.MaskedImage`, optional
1074 Differential background model
1076 Returns
1077 -------
1078 difference : `lsst.afw.Exposure`
1079 The subtracted image.
1080 """
1081 difference = science.clone()
1082 if backgroundModel is not None:
1083 difference.maskedImage -= backgroundModel
1084 difference.maskedImage -= template.maskedImage
1085 return difference
1088def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
1089 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
1091 Parameters
1092 ----------
1093 exp1 : `~lsst.afw.image.Exposure`
1094 Exposure with the reference point spread function (PSF) to evaluate.
1095 exp2 : `~lsst.afw.image.Exposure`
1096 Exposure with a candidate point spread function (PSF) to evaluate.
1097 fwhmExposureBuffer : `float`
1098 Fractional buffer margin to be left out of all sides of the image
1099 during the construction of the grid to compute mean PSF FWHM in an
1100 exposure, if the PSF is not available at its average position.
1101 fwhmExposureGrid : `int`
1102 Grid size to compute the mean FWHM in an exposure, if the PSF is not
1103 available at its average position.
1104 Returns
1105 -------
1106 result : `bool`
1107 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in
1108 either dimension.
1109 """
1110 try:
1111 shape1 = getPsfFwhm(exp1.psf, average=False)
1112 shape2 = getPsfFwhm(exp2.psf, average=False)
1113 except lsst.pex.exceptions.InvalidParameterError:
1114 shape1 = evaluateMeanPsfFwhm(exp1,
1115 fwhmExposureBuffer=fwhmExposureBuffer,
1116 fwhmExposureGrid=fwhmExposureGrid
1117 )
1118 shape2 = evaluateMeanPsfFwhm(exp2,
1119 fwhmExposureBuffer=fwhmExposureBuffer,
1120 fwhmExposureGrid=fwhmExposureGrid
1121 )
1122 return shape1 <= shape2
1124 # Results from getPsfFwhm is a tuple of two values, one for each dimension.
1125 xTest = shape1[0] <= shape2[0]
1126 yTest = shape1[1] <= shape2[1]
1127 return xTest | yTest
1130def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None):
1131 """Replace masked image pixels with interpolated values.
1133 Parameters
1134 ----------
1135 maskedImage : `lsst.afw.image.MaskedImage`
1136 Image on which to perform interpolation.
1137 badMaskPlanes : `list` of `str`
1138 List of mask planes to interpolate over.
1139 fallbackValue : `float`, optional
1140 Value to set when interpolation fails.
1142 Returns
1143 -------
1144 result: `float`
1145 The number of masked pixels that were replaced.
1146 """
1147 imgBadMaskPlanes = [
1148 maskPlane for maskPlane in badMaskPlanes if maskPlane in maskedImage.mask.getMaskPlaneDict()
1149 ]
1151 image = maskedImage.image.array
1152 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(imgBadMaskPlanes)) > 0
1153 image[badPixels] = np.nan
1154 if fallbackValue is None:
1155 fallbackValue = np.nanmedian(image)
1156 # For this initial implementation, skip the interpolation and just fill with
1157 # the median value.
1158 image[badPixels] = fallbackValue
1159 return np.sum(badPixels)