lsst.ip.diffim ga7c8634a61+caebfa2982
Loading...
Searching...
No Matches
subtractImages.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import warnings
23
24import numpy as np
25
26import lsst.afw.image
27import lsst.afw.math
28import lsst.geom
29from lsst.utils.introspection import find_outside_stacklevel
30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm
31from lsst.meas.algorithms import ScaleVarianceTask
32import lsst.pex.config
33import lsst.pipe.base
35from lsst.pipe.base import connectionTypes
36from . import MakeKernelTask, DecorrelateALKernelTask
37from lsst.utils.timer import timeMethod
38
39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask",
40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"]
41
42_dimensions = ("instrument", "visit", "detector")
43_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
44
45
46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
47 dimensions=_dimensions,
48 defaultTemplates=_defaultTemplates):
49 template = connectionTypes.Input(
50 doc="Input warped template to subtract.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}{coaddName}Diff_templateExp"
54 )
55 science = connectionTypes.Input(
56 doc="Input science exposure to subtract from.",
57 dimensions=("instrument", "visit", "detector"),
58 storageClass="ExposureF",
59 name="{fakesType}calexp"
60 )
61 sources = connectionTypes.Input(
62 doc="Sources measured on the science exposure; "
63 "used to select sources for making the matching kernel.",
64 dimensions=("instrument", "visit", "detector"),
65 storageClass="SourceCatalog",
66 name="{fakesType}src"
67 )
68 finalizedPsfApCorrCatalog = connectionTypes.Input(
69 doc=("Per-visit finalized psf models and aperture correction maps. "
70 "These catalogs use the detector id for the catalog id, "
71 "sorted on id for fast lookup."),
72 dimensions=("instrument", "visit"),
73 storageClass="ExposureCatalog",
74 name="finalVisitSummary",
75 # TODO: remove on DM-39854.
76 deprecated=(
77 "Deprecated in favor of visitSummary. Will be removed after v26."
78 )
79 )
80 visitSummary = connectionTypes.Input(
81 doc=("Per-visit catalog with final calibration objects. "
82 "These catalogs use the detector id for the catalog id, "
83 "sorted on id for fast lookup."),
84 dimensions=("instrument", "visit"),
85 storageClass="ExposureCatalog",
86 name="finalVisitSummary",
87 )
88
89 def __init__(self, *, config=None):
90 super().__init__(config=config)
91 if not config.doApplyFinalizedPsf:
92 self.inputs.remove("finalizedPsfApCorrCatalog")
93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf:
94 del self.visitSummary
95
96
97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
98 dimensions=_dimensions,
99 defaultTemplates=_defaultTemplates):
100 difference = connectionTypes.Output(
101 doc="Result of subtracting convolved template from science image.",
102 dimensions=("instrument", "visit", "detector"),
103 storageClass="ExposureF",
104 name="{fakesType}{coaddName}Diff_differenceTempExp",
105 )
106 matchedTemplate = connectionTypes.Output(
107 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
108 dimensions=("instrument", "visit", "detector"),
109 storageClass="ExposureF",
110 name="{fakesType}{coaddName}Diff_matchedExp",
111 )
112
113
114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections,
115 dimensions=_dimensions,
116 defaultTemplates=_defaultTemplates):
117 scoreExposure = connectionTypes.Output(
118 doc="The maximum likelihood image, used for the detection of diaSources.",
119 dimensions=("instrument", "visit", "detector"),
120 storageClass="ExposureF",
121 name="{fakesType}{coaddName}Diff_scoreExp",
122 )
123
124
129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
130 makeKernel = lsst.pex.config.ConfigurableField(
131 target=MakeKernelTask,
132 doc="Task to construct a matching kernel for convolution.",
133 )
134 doDecorrelation = lsst.pex.config.Field(
135 dtype=bool,
136 default=True,
137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
138 "kernel convolution? If True, also update the diffim PSF."
139 )
140 decorrelate = lsst.pex.config.ConfigurableField(
141 target=DecorrelateALKernelTask,
142 doc="Task to decorrelate the image difference.",
143 )
144 requiredTemplateFraction = lsst.pex.config.Field(
145 dtype=float,
146 default=0.1,
147 doc="Raise NoWorkFound and do not attempt image subtraction if template covers less than this "
148 " fraction of pixels. Setting to 0 will always attempt image subtraction."
149 )
150 minTemplateFractionForExpectedSuccess = lsst.pex.config.Field(
151 dtype=float,
152 default=0.2,
153 doc="Raise NoWorkFound if PSF-matching fails and template covers less than this fraction of pixels."
154 " If the fraction of pixels covered by the template is less than this value (and greater than"
155 " requiredTemplateFraction) this task is attempted but failure is anticipated and tolerated."
156 )
157 doScaleVariance = lsst.pex.config.Field(
158 dtype=bool,
159 default=True,
160 doc="Scale variance of the image difference?"
161 )
162 scaleVariance = lsst.pex.config.ConfigurableField(
163 target=ScaleVarianceTask,
164 doc="Subtask to rescale the variance of the template to the statistically expected level."
165 )
166 doSubtractBackground = lsst.pex.config.Field(
167 doc="Subtract the background fit when solving the kernel?",
168 dtype=bool,
169 default=True,
170 )
171 doApplyFinalizedPsf = lsst.pex.config.Field(
172 doc="Replace science Exposure's psf and aperture correction map"
173 " with those in finalizedPsfApCorrCatalog.",
174 dtype=bool,
175 default=False,
176 # TODO: remove on DM-39854.
177 deprecated=(
178 "Deprecated in favor of doApplyExternalCalibrations. "
179 "Will be removed after v26."
180 )
181 )
182 doApplyExternalCalibrations = lsst.pex.config.Field(
183 doc=(
184 "Replace science Exposure's calibration objects with those"
185 " in visitSummary. Ignored if `doApplyFinalizedPsf is True."
186 ),
187 dtype=bool,
188 default=False,
189 )
190 detectionThreshold = lsst.pex.config.Field(
191 dtype=float,
192 default=10,
193 doc="Minimum signal to noise ratio of detected sources "
194 "to use for calculating the PSF matching kernel."
195 )
196 badSourceFlags = lsst.pex.config.ListField(
197 dtype=str,
198 doc="Flags that, if set, the associated source should not "
199 "be used to determine the PSF matching kernel.",
200 default=("sky_source", "slot_Centroid_flag",
201 "slot_ApFlux_flag", "slot_PsfFlux_flag", ),
202 )
203 badMaskPlanes = lsst.pex.config.ListField(
204 dtype=str,
205 default=("NO_DATA", "BAD", "SAT", "EDGE", "FAKE"),
206 doc="Mask planes to exclude when selecting sources for PSF matching."
207 )
208 preserveTemplateMask = lsst.pex.config.ListField(
209 dtype=str,
210 default=("NO_DATA", "BAD", "SAT", "FAKE", "INJECTED", "INJECTED_CORE"),
211 doc="Mask planes from the template to propagate to the image difference."
212 )
213
214 def setDefaults(self):
215 self.makeKernel.kernel.name = "AL"
216 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground
217 self.makeKernel.kernel.active.spatialKernelOrder = 1
218 self.makeKernel.kernel.active.spatialBgOrder = 2
219
220
221class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
222 pipelineConnections=AlardLuptonSubtractConnections):
223 mode = lsst.pex.config.ChoiceField(
224 dtype=str,
225 default="convolveTemplate",
226 allowed={"auto": "Choose which image to convolve at runtime.",
227 "convolveScience": "Only convolve the science image.",
228 "convolveTemplate": "Only convolve the template image."},
229 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
230 )
231
232
233class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
234 """Compute the image difference of a science and template image using
235 the Alard & Lupton (1998) algorithm.
236 """
237 ConfigClass = AlardLuptonSubtractConfig
238 _DefaultName = "alardLuptonSubtract"
239
240 def __init__(self, **kwargs):
241 super().__init__(**kwargs)
242 self.makeSubtask("decorrelate")
243 self.makeSubtask("makeKernel")
244 if self.config.doScaleVariance:
245 self.makeSubtask("scaleVariance")
246
248 # Normalization is an extra, unnecessary, calculation and will result
249 # in mis-subtraction of the images if there are calibration errors.
250 self.convolutionControl.setDoNormalize(False)
251 self.convolutionControl.setDoCopyEdge(True)
252
253 def _applyExternalCalibrations(self, exposure, visitSummary):
254 """Replace calibrations (psf, and ApCorrMap) on this exposure with
255 external ones.".
256
257 Parameters
258 ----------
259 exposure : `lsst.afw.image.exposure.Exposure`
260 Input exposure to adjust calibrations.
261 visitSummary : `lsst.afw.table.ExposureCatalog`
262 Exposure catalog with external calibrations to be applied. Catalog
263 uses the detector id for the catalog id, sorted on id for fast
264 lookup.
265
266 Returns
267 -------
268 exposure : `lsst.afw.image.exposure.Exposure`
269 Exposure with adjusted calibrations.
270 """
271 detectorId = exposure.info.getDetector().getId()
272
273 row = visitSummary.find(detectorId)
274 if row is None:
275 self.log.warning("Detector id %s not found in external calibrations catalog; "
276 "Using original calibrations.", detectorId)
277 else:
278 psf = row.getPsf()
279 apCorrMap = row.getApCorrMap()
280 if psf is None:
281 self.log.warning("Detector id %s has None for psf in "
282 "external calibrations catalog; Using original psf and aperture correction.",
283 detectorId)
284 elif apCorrMap is None:
285 self.log.warning("Detector id %s has None for apCorrMap in "
286 "external calibrations catalog; Using original psf and aperture correction.",
287 detectorId)
288 else:
289 exposure.setPsf(psf)
290 exposure.info.setApCorrMap(apCorrMap)
291
292 return exposure
293
294 @timeMethod
295 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None,
296 visitSummary=None):
297 """PSF match, subtract, and decorrelate two images.
298
299 Parameters
300 ----------
301 template : `lsst.afw.image.ExposureF`
302 Template exposure, warped to match the science exposure.
303 science : `lsst.afw.image.ExposureF`
304 Science exposure to subtract from the template.
305 sources : `lsst.afw.table.SourceCatalog`
306 Identified sources on the science exposure. This catalog is used to
307 select sources in order to perform the AL PSF matching on stamp
308 images around them.
309 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
310 Exposure catalog with finalized psf models and aperture correction
311 maps to be applied. Catalog uses the detector id for the catalog
312 id, sorted on id for fast lookup. Deprecated in favor of
313 ``visitSummary``, and will be removed after v26.
314 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
315 Exposure catalog with external calibrations to be applied. Catalog
316 uses the detector id for the catalog id, sorted on id for fast
317 lookup. Ignored (for temporary backwards compatibility) if
318 ``finalizedPsfApCorrCatalog`` is provided.
319
320 Returns
321 -------
322 results : `lsst.pipe.base.Struct`
323 ``difference`` : `lsst.afw.image.ExposureF`
324 Result of subtracting template and science.
325 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
326 Warped and PSF-matched template exposure.
327 ``backgroundModel`` : `lsst.afw.math.Function2D`
328 Background model that was fit while solving for the
329 PSF-matching kernel
330 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
331 Kernel used to PSF-match the convolved image.
332
333 Raises
334 ------
335 RuntimeError
336 If an unsupported convolution mode is supplied.
337 RuntimeError
338 If there are too few sources to calculate the PSF matching kernel.
339 lsst.pipe.base.NoWorkFound
340 Raised if fraction of good pixels, defined as not having NO_DATA
341 set, is less then the configured requiredTemplateFraction
342 """
343
344 if finalizedPsfApCorrCatalog is not None:
345 warnings.warn(
346 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
347 "argument, and will be removed after v26.",
348 FutureWarning,
349 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
350 )
351 visitSummary = finalizedPsfApCorrCatalog
352
353 self._prepareInputs(template, science, visitSummary=visitSummary)
354
355 # In the event that getPsfFwhm fails, evaluate the PSF on a grid.
356 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
357 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
358
359 # Calling getPsfFwhm on template.psf fails on some rare occasions when
360 # the template has no input exposures at the average position of the
361 # stars. So we try getPsfFwhm first on template, and if that fails we
362 # evaluate the PSF on a grid specified by fwhmExposure* fields.
363 # To keep consistent definitions for PSF size on the template and
364 # science images, we use the same method for both.
365 try:
366 templatePsfSize = getPsfFwhm(template.psf)
367 sciencePsfSize = getPsfFwhm(science.psf)
369 self.log.info("Unable to evaluate PSF at the average position. "
370 "Evaluting PSF on a grid of points."
371 )
372 templatePsfSize = evaluateMeanPsfFwhm(template,
373 fwhmExposureBuffer=fwhmExposureBuffer,
374 fwhmExposureGrid=fwhmExposureGrid
375 )
376 sciencePsfSize = evaluateMeanPsfFwhm(science,
377 fwhmExposureBuffer=fwhmExposureBuffer,
378 fwhmExposureGrid=fwhmExposureGrid
379 )
380 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize)
381 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize)
382 self.metadata.add("sciencePsfSize", sciencePsfSize)
383 self.metadata.add("templatePsfSize", templatePsfSize)
384 selectSources = self._sourceSelector(sources, science.mask)
385
386 if self.config.mode == "auto":
387 convolveTemplate = _shapeTest(template,
388 science,
389 fwhmExposureBuffer=fwhmExposureBuffer,
390 fwhmExposureGrid=fwhmExposureGrid)
391 if convolveTemplate:
392 if sciencePsfSize < templatePsfSize:
393 self.log.info("Average template PSF size is greater, "
394 "but science PSF greater in one dimension: convolving template image.")
395 else:
396 self.log.info("Science PSF size is greater: convolving template image.")
397 else:
398 self.log.info("Template PSF size is greater: convolving science image.")
399 elif self.config.mode == "convolveTemplate":
400 self.log.info("`convolveTemplate` is set: convolving template image.")
401 convolveTemplate = True
402 elif self.config.mode == "convolveScience":
403 self.log.info("`convolveScience` is set: convolving science image.")
404 convolveTemplate = False
405 else:
406 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
407
408 try:
409 if convolveTemplate:
410 self.metadata.add("convolvedExposure", "Template")
411 subtractResults = self.runConvolveTemplate(template, science, selectSources)
412 else:
413 self.metadata.add("convolvedExposure", "Science")
414 subtractResults = self.runConvolveScience(template, science, selectSources)
415
416 except (RuntimeError, lsst.pex.exceptions.Exception) as e:
417 self.log.warn("Failed to match template. Checking coverage")
418 # Raise NoWorkFound if template fraction is insufficient
420 self.config.minTemplateFractionForExpectedSuccess,
421 exceptionMessage="Template coverage lower than expected to succeed."
422 f" Failure is tolerable: {e}")
423 # checkTemplateIsSufficient did not raise NoWorkFound, so raise original exception
424 raise e
425
426 return subtractResults
427
428 def runConvolveTemplate(self, template, science, selectSources):
429 """Convolve the template image with a PSF-matching kernel and subtract
430 from the science image.
431
432 Parameters
433 ----------
434 template : `lsst.afw.image.ExposureF`
435 Template exposure, warped to match the science exposure.
436 science : `lsst.afw.image.ExposureF`
437 Science exposure to subtract from the template.
438 selectSources : `lsst.afw.table.SourceCatalog`
439 Identified sources on the science exposure. This catalog is used to
440 select sources in order to perform the AL PSF matching on stamp
441 images around them.
442
443 Returns
444 -------
445 results : `lsst.pipe.base.Struct`
446
447 ``difference`` : `lsst.afw.image.ExposureF`
448 Result of subtracting template and science.
449 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
450 Warped and PSF-matched template exposure.
451 ``backgroundModel`` : `lsst.afw.math.Function2D`
452 Background model that was fit while solving for the PSF-matching kernel
453 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
454 Kernel used to PSF-match the template to the science image.
455 """
456 kernelSources = self.makeKernel.selectKernelSources(template, science,
457 candidateList=selectSources,
458 preconvolved=False)
459 kernelResult = self.makeKernel.run(template, science, kernelSources,
460 preconvolved=False)
461
462 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
464 bbox=science.getBBox(),
465 psf=science.psf,
466 photoCalib=science.photoCalib)
467
468 difference = _subtractImages(science, matchedTemplate,
469 backgroundModel=(kernelResult.backgroundModel
470 if self.config.doSubtractBackground else None))
471 correctedExposure = self.finalize(template, science, difference,
472 kernelResult.psfMatchingKernel,
473 templateMatched=True)
474
475 return lsst.pipe.base.Struct(difference=correctedExposure,
476 matchedTemplate=matchedTemplate,
477 matchedScience=science,
478 backgroundModel=kernelResult.backgroundModel,
479 psfMatchingKernel=kernelResult.psfMatchingKernel)
480
481 def runConvolveScience(self, template, science, selectSources):
482 """Convolve the science image with a PSF-matching kernel and subtract
483 the template image.
484
485 Parameters
486 ----------
487 template : `lsst.afw.image.ExposureF`
488 Template exposure, warped to match the science exposure.
489 science : `lsst.afw.image.ExposureF`
490 Science exposure to subtract from the template.
491 selectSources : `lsst.afw.table.SourceCatalog`
492 Identified sources on the science exposure. This catalog is used to
493 select sources in order to perform the AL PSF matching on stamp
494 images around them.
495
496 Returns
497 -------
498 results : `lsst.pipe.base.Struct`
499
500 ``difference`` : `lsst.afw.image.ExposureF`
501 Result of subtracting template and science.
502 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
503 Warped template exposure. Note that in this case, the template
504 is not PSF-matched to the science image.
505 ``backgroundModel`` : `lsst.afw.math.Function2D`
506 Background model that was fit while solving for the PSF-matching kernel
507 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
508 Kernel used to PSF-match the science image to the template.
509 """
510 bbox = science.getBBox()
511 kernelSources = self.makeKernel.selectKernelSources(science, template,
512 candidateList=selectSources,
513 preconvolved=False)
514 kernelResult = self.makeKernel.run(science, template, kernelSources,
515 preconvolved=False)
516 modelParams = kernelResult.backgroundModel.getParameters()
517 # We must invert the background model if the matching kernel is solved for the science image.
518 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
519
520 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
521 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False)
522
523 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel,
525 psf=template.psf)
526
527 # Place back on native photometric scale
528 matchedScience.maskedImage /= norm
529 matchedTemplate = template.clone()[bbox]
530 matchedTemplate.maskedImage /= norm
531 matchedTemplate.setPhotoCalib(science.photoCalib)
532
533 difference = _subtractImages(matchedScience, matchedTemplate,
534 backgroundModel=(kernelResult.backgroundModel
535 if self.config.doSubtractBackground else None))
536
537 correctedExposure = self.finalize(template, science, difference,
538 kernelResult.psfMatchingKernel,
539 templateMatched=False)
540
541 return lsst.pipe.base.Struct(difference=correctedExposure,
542 matchedTemplate=matchedTemplate,
543 matchedScience=matchedScience,
544 backgroundModel=kernelResult.backgroundModel,
545 psfMatchingKernel=kernelResult.psfMatchingKernel,)
546
547 def finalize(self, template, science, difference, kernel,
548 templateMatched=True,
549 preConvMode=False,
550 preConvKernel=None,
551 spatiallyVarying=False):
552 """Decorrelate the difference image to undo the noise correlations
553 caused by convolution.
554
555 Parameters
556 ----------
557 template : `lsst.afw.image.ExposureF`
558 Template exposure, warped to match the science exposure.
559 science : `lsst.afw.image.ExposureF`
560 Science exposure to subtract from the template.
561 difference : `lsst.afw.image.ExposureF`
562 Result of subtracting template and science.
563 kernel : `lsst.afw.math.Kernel`
564 An (optionally spatially-varying) PSF matching kernel
565 templateMatched : `bool`, optional
566 Was the template PSF-matched to the science image?
567 preConvMode : `bool`, optional
568 Was the science image preconvolved with its own PSF
569 before PSF matching the template?
570 preConvKernel : `lsst.afw.detection.Psf`, optional
571 If not `None`, then the science image was pre-convolved with
572 (the reflection of) this kernel. Must be normalized to sum to 1.
573 spatiallyVarying : `bool`, optional
574 Compute the decorrelation kernel spatially varying across the image?
575
576 Returns
577 -------
578 correctedExposure : `lsst.afw.image.ExposureF`
579 The decorrelated image difference.
580 """
581 # Erase existing detection mask planes.
582 # We don't want the detection mask from the science image
583
584 self.updateMasks(template, science, difference)
585
586 if self.config.doDecorrelation:
587 self.log.info("Decorrelating image difference.")
588 # We have cleared the template mask plane, so copy the mask plane of
589 # the image difference so that we can calculate correct statistics
590 # during decorrelation
591 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
592 templateMatched=templateMatched,
593 preConvMode=preConvMode,
594 preConvKernel=preConvKernel,
595 spatiallyVarying=spatiallyVarying).correctedExposure
596 else:
597 self.log.info("NOT decorrelating image difference.")
598 correctedExposure = difference
599 return correctedExposure
600
601 def updateMasks(self, template, science, difference):
602 """Update the mask planes on images for finalizing."""
603
604 bbox = science.getBBox()
605 mask = difference.mask
606 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
607
608 if "FAKE" in science.mask.getMaskPlaneDict().keys():
609 # propagate the mask plane related to Fake source injection
610 # NOTE: the fake source injection sets FAKE plane, but it should be INJECTED
611 # NOTE: This can be removed in DM-40796
612
613 self.log.info("Adding injected mask planes")
614 mask.addMaskPlane("INJECTED")
615 diffInjectedBitMask = mask.getPlaneBitMask("INJECTED")
616
617 mask.addMaskPlane("INJECTED_TEMPLATE")
618 diffInjTmpltBitMask = mask.getPlaneBitMask("INJECTED_TEMPLATE")
619
620 scienceFakeBitMask = science.mask.getPlaneBitMask('FAKE')
621 tmpltFakeBitMask = template[bbox].mask.getPlaneBitMask('FAKE')
622
623 injScienceMaskArray = ((science.mask.array & scienceFakeBitMask) > 0) * diffInjectedBitMask
624 injTemplateMaskArray = ((template[bbox].mask.array & tmpltFakeBitMask) > 0) * diffInjTmpltBitMask
625
626 mask.array |= injScienceMaskArray
627 mask.array |= injTemplateMaskArray
628
629 template[bbox].mask.array[...] = difference.mask.array[...]
630
631 @staticmethod
632 def _validateExposures(template, science):
633 """Check that the WCS of the two Exposures match, and the template bbox
634 contains the science bbox.
635
636 Parameters
637 ----------
638 template : `lsst.afw.image.ExposureF`
639 Template exposure, warped to match the science exposure.
640 science : `lsst.afw.image.ExposureF`
641 Science exposure to subtract from the template.
642
643 Raises
644 ------
645 AssertionError
646 Raised if the WCS of the template is not equal to the science WCS,
647 or if the science image is not fully contained in the template
648 bounding box.
649 """
650 assert template.wcs == science.wcs,\
651 "Template and science exposure WCS are not identical."
652 templateBBox = template.getBBox()
653 scienceBBox = science.getBBox()
654
655 assert templateBBox.contains(scienceBBox),\
656 "Template bbox does not contain all of the science image."
657
658 def _convolveExposure(self, exposure, kernel, convolutionControl,
659 bbox=None,
660 psf=None,
661 photoCalib=None,
662 interpolateBadMaskPlanes=False,
663 ):
664 """Convolve an exposure with the given kernel.
665
666 Parameters
667 ----------
668 exposure : `lsst.afw.Exposure`
669 exposure to convolve.
670 kernel : `lsst.afw.math.LinearCombinationKernel`
671 PSF matching kernel computed in the ``makeKernel`` subtask.
672 convolutionControl : `lsst.afw.math.ConvolutionControl`
673 Configuration for convolve algorithm.
674 bbox : `lsst.geom.Box2I`, optional
675 Bounding box to trim the convolved exposure to.
676 psf : `lsst.afw.detection.Psf`, optional
677 Point spread function (PSF) to set for the convolved exposure.
678 photoCalib : `lsst.afw.image.PhotoCalib`, optional
679 Photometric calibration of the convolved exposure.
680
681 Returns
682 -------
683 convolvedExp : `lsst.afw.Exposure`
684 The convolved image.
685 """
686 convolvedExposure = exposure.clone()
687 if psf is not None:
688 convolvedExposure.setPsf(psf)
689 if photoCalib is not None:
690 convolvedExposure.setPhotoCalib(photoCalib)
691 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None:
692 nInterp = _interpolateImage(convolvedExposure.maskedImage,
693 self.config.badMaskPlanes)
694 self.metadata.add("nInterpolated", nInterp)
695 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox())
696 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl)
697 convolvedExposure.setMaskedImage(convolvedImage)
698 if bbox is None:
699 return convolvedExposure
700 else:
701 return convolvedExposure[bbox]
702
703 def _sourceSelector(self, sources, mask):
704 """Select sources from a catalog that meet the selection criteria.
705
706 Parameters
707 ----------
708 sources : `lsst.afw.table.SourceCatalog`
709 Input source catalog to select sources from.
710 mask : `lsst.afw.image.Mask`
711 The image mask plane to use to reject sources
712 based on their location on the ccd.
713
714 Returns
715 -------
716 selectSources : `lsst.afw.table.SourceCatalog`
717 The input source catalog, with flagged and low signal-to-noise
718 sources removed.
719
720 Raises
721 ------
722 RuntimeError
723 If there are too few sources to compute the PSF matching kernel
724 remaining after source selection.
725 """
726 flags = np.ones(len(sources), dtype=bool)
727 for flag in self.config.badSourceFlags:
728 try:
729 flags *= ~sources[flag]
730 except Exception as e:
731 self.log.warning("Could not apply source flag: %s", e)
732 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
733 flags *= sToNFlag
734 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes)
735 selectSources = sources[flags]
736 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog",
737 len(selectSources), len(sources), 100*len(selectSources)/len(sources))
738 if len(selectSources) < self.config.makeKernel.nStarPerCell:
739 self.log.error("Too few sources to calculate the PSF matching kernel: "
740 "%i selected but %i needed for the calculation.",
741 len(selectSources), self.config.makeKernel.nStarPerCell)
742 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.")
743 self.metadata.add("nPsfSources", len(selectSources))
744
745 return selectSources.copy(deep=True)
746
747 @staticmethod
748 def _checkMask(mask, sources, badMaskPlanes):
749 """Exclude sources that are located on masked pixels.
750
751 Parameters
752 ----------
753 mask : `lsst.afw.image.Mask`
754 The image mask plane to use to reject sources
755 based on the location of their centroid on the ccd.
756 sources : `lsst.afw.table.SourceCatalog`
757 The source catalog to evaluate.
758 badMaskPlanes : `list` of `str`
759 List of the names of the mask planes to exclude.
760
761 Returns
762 -------
763 flags : `numpy.ndarray` of `bool`
764 Array indicating whether each source in the catalog should be
765 kept (True) or rejected (False) based on the value of the
766 mask plane at its location.
767 """
768 setBadMaskPlanes = [
769 maskPlane for maskPlane in badMaskPlanes if maskPlane in mask.getMaskPlaneDict()
770 ]
771
772 badPixelMask = mask.getPlaneBitMask(setBadMaskPlanes)
773
774 xv = np.rint(sources.getX() - mask.getX0())
775 yv = np.rint(sources.getY() - mask.getY0())
776
777 mv = mask.array[yv.astype(int), xv.astype(int)]
778 flags = np.bitwise_and(mv, badPixelMask) == 0
779 return flags
780
781 def _prepareInputs(self, template, science, visitSummary=None):
782 """Perform preparatory calculations common to all Alard&Lupton Tasks.
783
784 Parameters
785 ----------
786 template : `lsst.afw.image.ExposureF`
787 Template exposure, warped to match the science exposure. The
788 variance plane of the template image is modified in place.
789 science : `lsst.afw.image.ExposureF`
790 Science exposure to subtract from the template. The variance plane
791 of the science image is modified in place.
792 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
793 Exposure catalog with external calibrations to be applied. Catalog
794 uses the detector id for the catalog id, sorted on id for fast
795 lookup.
796 """
797 self._validateExposures(template, science)
798 if visitSummary is not None:
799 self._applyExternalCalibrations(science, visitSummary=visitSummary)
800 checkTemplateIsSufficient(template, self.log,
801 requiredTemplateFraction=self.config.requiredTemplateFraction,
802 exceptionMessage="Not attempting subtraction. To force subtraction,"
803 " set config requiredTemplateFraction=0")
804
805 if self.config.doScaleVariance:
806 # Scale the variance of the template and science images before
807 # convolution, subtraction, or decorrelation so that they have the
808 # correct ratio.
809 templateVarFactor = self.scaleVariance.run(template.maskedImage)
810 sciVarFactor = self.scaleVariance.run(science.maskedImage)
811 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
812 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
813 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
814 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor)
815 self._clearMask(template)
816
817 def _clearMask(self, template):
818 """Clear the mask plane of the template.
819
820 Parameters
821 ----------
822 template : `lsst.afw.image.ExposureF`
823 Template exposure, warped to match the science exposure.
824 The mask plane will be modified in place.
825 """
826 mask = template.mask
827 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys()
828 if maskplane not in self.config.preserveTemplateMask]
829
830 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes)
831 mask &= ~bitMaskToClear
832
833
835 SubtractScoreOutputConnections):
836 pass
837
838
840 pipelineConnections=AlardLuptonPreconvolveSubtractConnections):
841 pass
842
843
845 """Subtract a template from a science image, convolving the science image
846 before computing the kernel, and also convolving the template before
847 subtraction.
848 """
849 ConfigClass = AlardLuptonPreconvolveSubtractConfig
850 _DefaultName = "alardLuptonPreconvolveSubtract"
851
852 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
853 """Preconvolve the science image with its own PSF,
854 convolve the template image with a PSF-matching kernel and subtract
855 from the preconvolved science image.
856
857 Parameters
858 ----------
859 template : `lsst.afw.image.ExposureF`
860 The template image, which has previously been warped to the science
861 image. The template bbox will be padded by a few pixels compared to
862 the science bbox.
863 science : `lsst.afw.image.ExposureF`
864 The science exposure.
865 sources : `lsst.afw.table.SourceCatalog`
866 Identified sources on the science exposure. This catalog is used to
867 select sources in order to perform the AL PSF matching on stamp
868 images around them.
869 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
870 Exposure catalog with finalized psf models and aperture correction
871 maps to be applied. Catalog uses the detector id for the catalog
872 id, sorted on id for fast lookup. Deprecated in favor of
873 ``visitSummary``, and will be removed after v26.
874 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
875 Exposure catalog with complete external calibrations. Catalog uses
876 the detector id for the catalog id, sorted on id for fast lookup.
877 Ignored (for temporary backwards compatibility) if
878 ``finalizedPsfApCorrCatalog`` is provided.
879
880 Returns
881 -------
882 results : `lsst.pipe.base.Struct`
883 ``scoreExposure`` : `lsst.afw.image.ExposureF`
884 Result of subtracting the convolved template and science
885 images. Attached PSF is that of the original science image.
886 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
887 Warped and PSF-matched template exposure. Attached PSF is that
888 of the original science image.
889 ``matchedScience`` : `lsst.afw.image.ExposureF`
890 The science exposure after convolving with its own PSF.
891 Attached PSF is that of the original science image.
892 ``backgroundModel`` : `lsst.afw.math.Function2D`
893 Background model that was fit while solving for the
894 PSF-matching kernel
895 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
896 Final kernel used to PSF-match the template to the science
897 image.
898 """
899 if finalizedPsfApCorrCatalog is not None:
900 warnings.warn(
901 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
902 "argument, and will be removed after v26.",
903 FutureWarning,
904 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
905 )
906 visitSummary = finalizedPsfApCorrCatalog
907
908 self._prepareInputs(template, science, visitSummary=visitSummary)
909
910 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation
911 scienceKernel = science.psf.getKernel()
912 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControlconvolutionControl,
913 interpolateBadMaskPlanes=True)
914 selectSources = self._sourceSelector(sources, matchedScience.mask)
915 self.metadata.add("convolvedExposure", "Preconvolution")
916
917 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel)
918
919 return subtractResults
920
921 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel):
922 """Convolve the science image with its own PSF, then convolve the
923 template with a matching kernel and subtract to form the Score
924 exposure.
925
926 Parameters
927 ----------
928 template : `lsst.afw.image.ExposureF`
929 Template exposure, warped to match the science exposure.
930 science : `lsst.afw.image.ExposureF`
931 Science exposure to subtract from the template.
932 matchedScience : `lsst.afw.image.ExposureF`
933 The science exposure, convolved with the reflection of its own PSF.
934 selectSources : `lsst.afw.table.SourceCatalog`
935 Identified sources on the science exposure. This catalog is used to
936 select sources in order to perform the AL PSF matching on stamp
937 images around them.
938 preConvKernel : `lsst.afw.math.Kernel`
939 The reflection of the kernel that was used to preconvolve the
940 `science` exposure. Must be normalized to sum to 1.
941
942 Returns
943 -------
944 results : `lsst.pipe.base.Struct`
945
946 ``scoreExposure`` : `lsst.afw.image.ExposureF`
947 Result of subtracting the convolved template and science
948 images. Attached PSF is that of the original science image.
949 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
950 Warped and PSF-matched template exposure. Attached PSF is that
951 of the original science image.
952 ``matchedScience`` : `lsst.afw.image.ExposureF`
953 The science exposure after convolving with its own PSF.
954 Attached PSF is that of the original science image.
955 ``backgroundModel`` : `lsst.afw.math.Function2D`
956 Background model that was fit while solving for the
957 PSF-matching kernel
958 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
959 Final kernel used to PSF-match the template to the science
960 image.
961 """
962 bbox = science.getBBox()
963 innerBBox = preConvKernel.shrinkBBox(bbox)
964
965 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox],
966 candidateList=selectSources,
967 preconvolved=True)
968 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources,
969 preconvolved=True)
970
971 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
973 bbox=bbox,
974 psf=science.psf,
975 interpolateBadMaskPlanes=True,
976 photoCalib=science.photoCalib)
977 score = _subtractImages(matchedScience, matchedTemplate,
978 backgroundModel=(kernelResult.backgroundModel
979 if self.config.doSubtractBackground else None))
980 correctedScore = self.finalize(template[bbox], science, score,
981 kernelResult.psfMatchingKernel,
982 templateMatched=True, preConvMode=True,
983 preConvKernel=preConvKernel)
984
985 return lsst.pipe.base.Struct(scoreExposure=correctedScore,
986 matchedTemplate=matchedTemplate,
987 matchedScience=matchedScience,
988 backgroundModel=kernelResult.backgroundModel,
989 psfMatchingKernel=kernelResult.psfMatchingKernel)
990
991
992def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.,
993 exceptionMessage=""):
994 """Raise NoWorkFound if template coverage < requiredTemplateFraction
995
996 Parameters
997 ----------
998 templateExposure : `lsst.afw.image.ExposureF`
999 The template exposure to check
1000 logger : `lsst.log.Log`
1001 Logger for printing output.
1002 requiredTemplateFraction : `float`, optional
1003 Fraction of pixels of the science image required to have coverage
1004 in the template.
1005 exceptionMessage : `str`, optional
1006 Message to include in the exception raised if the template coverage
1007 is insufficient.
1008
1009 Raises
1010 ------
1011 lsst.pipe.base.NoWorkFound
1012 Raised if fraction of good pixels, defined as not having NO_DATA
1013 set, is less than the requiredTemplateFraction
1014 """
1015 # Count the number of pixels with the NO_DATA mask bit set
1016 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
1017 pixNoData = np.count_nonzero(templateExposure.mask.array
1018 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
1019 pixGood = templateExposure.getBBox().getArea() - pixNoData
1020 logger.info("template has %d good pixels (%.1f%%)", pixGood,
1021 100*pixGood/templateExposure.getBBox().getArea())
1022
1023 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
1024 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%)" % (
1025 100*pixGood/templateExposure.getBBox().getArea(),
1026 100*requiredTemplateFraction))
1027 raise lsst.pipe.base.NoWorkFound(message + " " + exceptionMessage)
1028
1029
1030def _subtractImages(science, template, backgroundModel=None):
1031 """Subtract template from science, propagating relevant metadata.
1032
1033 Parameters
1034 ----------
1035 science : `lsst.afw.Exposure`
1036 The input science image.
1037 template : `lsst.afw.Exposure`
1038 The template to subtract from the science image.
1039 backgroundModel : `lsst.afw.MaskedImage`, optional
1040 Differential background model
1041
1042 Returns
1043 -------
1044 difference : `lsst.afw.Exposure`
1045 The subtracted image.
1046 """
1047 difference = science.clone()
1048 if backgroundModel is not None:
1049 difference.maskedImage -= backgroundModel
1050 difference.maskedImage -= template.maskedImage
1051 return difference
1052
1053
1054def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
1055 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
1056
1057 Parameters
1058 ----------
1059 exp1 : `~lsst.afw.image.Exposure`
1060 Exposure with the reference point spread function (PSF) to evaluate.
1061 exp2 : `~lsst.afw.image.Exposure`
1062 Exposure with a candidate point spread function (PSF) to evaluate.
1063 fwhmExposureBuffer : `float`
1064 Fractional buffer margin to be left out of all sides of the image
1065 during the construction of the grid to compute mean PSF FWHM in an
1066 exposure, if the PSF is not available at its average position.
1067 fwhmExposureGrid : `int`
1068 Grid size to compute the mean FWHM in an exposure, if the PSF is not
1069 available at its average position.
1070 Returns
1071 -------
1072 result : `bool`
1073 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in
1074 either dimension.
1075 """
1076 try:
1077 shape1 = getPsfFwhm(exp1.psf, average=False)
1078 shape2 = getPsfFwhm(exp2.psf, average=False)
1080 shape1 = evaluateMeanPsfFwhm(exp1,
1081 fwhmExposureBuffer=fwhmExposureBuffer,
1082 fwhmExposureGrid=fwhmExposureGrid
1083 )
1084 shape2 = evaluateMeanPsfFwhm(exp2,
1085 fwhmExposureBuffer=fwhmExposureBuffer,
1086 fwhmExposureGrid=fwhmExposureGrid
1087 )
1088 return shape1 <= shape2
1089
1090 # Results from getPsfFwhm is a tuple of two values, one for each dimension.
1091 xTest = shape1[0] <= shape2[0]
1092 yTest = shape1[1] <= shape2[1]
1093 return xTest | yTest
1094
1095
1096def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None):
1097 """Replace masked image pixels with interpolated values.
1098
1099 Parameters
1100 ----------
1101 maskedImage : `lsst.afw.image.MaskedImage`
1102 Image on which to perform interpolation.
1103 badMaskPlanes : `list` of `str`
1104 List of mask planes to interpolate over.
1105 fallbackValue : `float`, optional
1106 Value to set when interpolation fails.
1107
1108 Returns
1109 -------
1110 result: `float`
1111 The number of masked pixels that were replaced.
1112 """
1113 imgBadMaskPlanes = [
1114 maskPlane for maskPlane in badMaskPlanes if maskPlane in maskedImage.mask.getMaskPlaneDict()
1115 ]
1116
1117 image = maskedImage.image.array
1118 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(imgBadMaskPlanes)) > 0
1119 image[badPixels] = np.nan
1120 if fallbackValue is None:
1121 fallbackValue = np.nanmedian(image)
1122 # For this initial implementation, skip the interpolation and just fill with
1123 # the median value.
1124 image[badPixels] = fallbackValue
1125 return np.sum(badPixels)
Asseses the quality of a candidate given a spatial kernel and background model.
runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel)
run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None)
_prepareInputs(self, template, science, visitSummary=None)
runConvolveTemplate(self, template, science, selectSources)
_convolveExposure(self, exposure, kernel, convolutionControl, bbox=None, psf=None, photoCalib=None, interpolateBadMaskPlanes=False)
runConvolveScience(self, template, science, selectSources)
updateMasks(self, template, science, difference)
run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None)
finalize(self, template, science, difference, kernel, templateMatched=True, preConvMode=False, preConvKernel=None, spatiallyVarying=False)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, ConvolutionControl const &convolutionControl=ConvolutionControl())
_subtractImages(science, template, backgroundModel=None)
_interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None)
checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0., exceptionMessage="")
_shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid)