lsst.ip.diffim g21e1ef4493+7ea36691c6
Loading...
Searching...
No Matches
subtractImages.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import warnings
23
24import numpy as np
25
26import lsst.afw.image
27import lsst.afw.math
28import lsst.geom
29from lsst.utils.introspection import find_outside_stacklevel
30from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm
31from lsst.meas.algorithms import ScaleVarianceTask
32import lsst.pex.config
33import lsst.pipe.base
34from lsst.pex.exceptions import InvalidParameterError
35from lsst.pipe.base import connectionTypes
36from . import MakeKernelTask, DecorrelateALKernelTask
37from lsst.utils.timer import timeMethod
38
39__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask",
40 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"]
41
42_dimensions = ("instrument", "visit", "detector")
43_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
44
45
46class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
47 dimensions=_dimensions,
48 defaultTemplates=_defaultTemplates):
49 template = connectionTypes.Input(
50 doc="Input warped template to subtract.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}{coaddName}Diff_templateExp"
54 )
55 science = connectionTypes.Input(
56 doc="Input science exposure to subtract from.",
57 dimensions=("instrument", "visit", "detector"),
58 storageClass="ExposureF",
59 name="{fakesType}calexp"
60 )
61 sources = connectionTypes.Input(
62 doc="Sources measured on the science exposure; "
63 "used to select sources for making the matching kernel.",
64 dimensions=("instrument", "visit", "detector"),
65 storageClass="SourceCatalog",
66 name="{fakesType}src"
67 )
68 finalizedPsfApCorrCatalog = connectionTypes.Input(
69 doc=("Per-visit finalized psf models and aperture correction maps. "
70 "These catalogs use the detector id for the catalog id, "
71 "sorted on id for fast lookup."),
72 dimensions=("instrument", "visit"),
73 storageClass="ExposureCatalog",
74 name="finalVisitSummary",
75 # TODO: remove on DM-39854.
76 deprecated=(
77 "Deprecated in favor of visitSummary. Will be removed after v26."
78 )
79 )
80 visitSummary = connectionTypes.Input(
81 doc=("Per-visit catalog with final calibration objects. "
82 "These catalogs use the detector id for the catalog id, "
83 "sorted on id for fast lookup."),
84 dimensions=("instrument", "visit"),
85 storageClass="ExposureCatalog",
86 name="finalVisitSummary",
87 )
88
89 def __init__(self, *, config=None):
90 super().__init__(config=config)
91 if not config.doApplyFinalizedPsf:
92 self.inputs.remove("finalizedPsfApCorrCatalog")
93 if not config.doApplyExternalCalibrations or config.doApplyFinalizedPsf:
94 del self.visitSummary
95
96
97class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
98 dimensions=_dimensions,
99 defaultTemplates=_defaultTemplates):
100 difference = connectionTypes.Output(
101 doc="Result of subtracting convolved template from science image.",
102 dimensions=("instrument", "visit", "detector"),
103 storageClass="ExposureF",
104 name="{fakesType}{coaddName}Diff_differenceTempExp",
105 )
106 matchedTemplate = connectionTypes.Output(
107 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
108 dimensions=("instrument", "visit", "detector"),
109 storageClass="ExposureF",
110 name="{fakesType}{coaddName}Diff_matchedExp",
111 )
112
113
114class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections,
115 dimensions=_dimensions,
116 defaultTemplates=_defaultTemplates):
117 scoreExposure = connectionTypes.Output(
118 doc="The maximum likelihood image, used for the detection of diaSources.",
119 dimensions=("instrument", "visit", "detector"),
120 storageClass="ExposureF",
121 name="{fakesType}{coaddName}Diff_scoreExp",
122 )
123
124
129class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
130 makeKernel = lsst.pex.config.ConfigurableField(
131 target=MakeKernelTask,
132 doc="Task to construct a matching kernel for convolution.",
133 )
134 doDecorrelation = lsst.pex.config.Field(
135 dtype=bool,
136 default=True,
137 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
138 "kernel convolution? If True, also update the diffim PSF."
139 )
140 decorrelate = lsst.pex.config.ConfigurableField(
141 target=DecorrelateALKernelTask,
142 doc="Task to decorrelate the image difference.",
143 )
144 requiredTemplateFraction = lsst.pex.config.Field(
145 dtype=float,
146 default=0.1,
147 doc="Abort task if template covers less than this fraction of pixels."
148 " Setting to 0 will always attempt image subtraction."
149 )
150 doScaleVariance = lsst.pex.config.Field(
151 dtype=bool,
152 default=True,
153 doc="Scale variance of the image difference?"
154 )
155 scaleVariance = lsst.pex.config.ConfigurableField(
156 target=ScaleVarianceTask,
157 doc="Subtask to rescale the variance of the template to the statistically expected level."
158 )
159 doSubtractBackground = lsst.pex.config.Field(
160 doc="Subtract the background fit when solving the kernel?",
161 dtype=bool,
162 default=True,
163 )
164 doApplyFinalizedPsf = lsst.pex.config.Field(
165 doc="Replace science Exposure's psf and aperture correction map"
166 " with those in finalizedPsfApCorrCatalog.",
167 dtype=bool,
168 default=False,
169 # TODO: remove on DM-39854.
170 deprecated=(
171 "Deprecated in favor of doApplyExternalCalibrations. "
172 "Will be removed after v26."
173 )
174 )
175 doApplyExternalCalibrations = lsst.pex.config.Field(
176 doc=(
177 "Replace science Exposure's calibration objects with those"
178 " in visitSummary. Ignored if `doApplyFinalizedPsf is True."
179 ),
180 dtype=bool,
181 default=False,
182 )
183 detectionThreshold = lsst.pex.config.Field(
184 dtype=float,
185 default=10,
186 doc="Minimum signal to noise ratio of detected sources "
187 "to use for calculating the PSF matching kernel."
188 )
189 badSourceFlags = lsst.pex.config.ListField(
190 dtype=str,
191 doc="Flags that, if set, the associated source should not "
192 "be used to determine the PSF matching kernel.",
193 default=("sky_source", "slot_Centroid_flag",
194 "slot_ApFlux_flag", "slot_PsfFlux_flag", ),
195 )
196 badMaskPlanes = lsst.pex.config.ListField(
197 dtype=str,
198 default=("NO_DATA", "BAD", "SAT", "EDGE", "FAKE"),
199 doc="Mask planes to exclude when selecting sources for PSF matching."
200 )
201 preserveTemplateMask = lsst.pex.config.ListField(
202 dtype=str,
203 default=("NO_DATA", "BAD", "SAT", "FAKE", "INJECTED", "INJECTED_CORE"),
204 doc="Mask planes from the template to propagate to the image difference."
205 )
206
207 def setDefaults(self):
208 self.makeKernel.kernel.name = "AL"
209 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground
210 self.makeKernel.kernel.active.spatialKernelOrder = 1
211 self.makeKernel.kernel.active.spatialBgOrder = 2
212
213
214class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
215 pipelineConnections=AlardLuptonSubtractConnections):
216 mode = lsst.pex.config.ChoiceField(
217 dtype=str,
218 default="convolveTemplate",
219 allowed={"auto": "Choose which image to convolve at runtime.",
220 "convolveScience": "Only convolve the science image.",
221 "convolveTemplate": "Only convolve the template image."},
222 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
223 )
224
225
226class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
227 """Compute the image difference of a science and template image using
228 the Alard & Lupton (1998) algorithm.
229 """
230 ConfigClass = AlardLuptonSubtractConfig
231 _DefaultName = "alardLuptonSubtract"
232
233 def __init__(self, **kwargs):
234 super().__init__(**kwargs)
235 self.makeSubtask("decorrelate")
236 self.makeSubtask("makeKernel")
237 if self.config.doScaleVariance:
238 self.makeSubtask("scaleVariance")
239
241 # Normalization is an extra, unnecessary, calculation and will result
242 # in mis-subtraction of the images if there are calibration errors.
243 self.convolutionControl.setDoNormalize(False)
244 self.convolutionControl.setDoCopyEdge(True)
245
246 def _applyExternalCalibrations(self, exposure, visitSummary):
247 """Replace calibrations (psf, and ApCorrMap) on this exposure with
248 external ones.".
249
250 Parameters
251 ----------
252 exposure : `lsst.afw.image.exposure.Exposure`
253 Input exposure to adjust calibrations.
254 visitSummary : `lsst.afw.table.ExposureCatalog`
255 Exposure catalog with external calibrations to be applied. Catalog
256 uses the detector id for the catalog id, sorted on id for fast
257 lookup.
258
259 Returns
260 -------
261 exposure : `lsst.afw.image.exposure.Exposure`
262 Exposure with adjusted calibrations.
263 """
264 detectorId = exposure.info.getDetector().getId()
265
266 row = visitSummary.find(detectorId)
267 if row is None:
268 self.log.warning("Detector id %s not found in external calibrations catalog; "
269 "Using original calibrations.", detectorId)
270 else:
271 psf = row.getPsf()
272 apCorrMap = row.getApCorrMap()
273 if psf is None:
274 self.log.warning("Detector id %s has None for psf in "
275 "external calibrations catalog; Using original psf and aperture correction.",
276 detectorId)
277 elif apCorrMap is None:
278 self.log.warning("Detector id %s has None for apCorrMap in "
279 "external calibrations catalog; Using original psf and aperture correction.",
280 detectorId)
281 else:
282 exposure.setPsf(psf)
283 exposure.info.setApCorrMap(apCorrMap)
284
285 return exposure
286
287 @timeMethod
288 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None,
289 visitSummary=None):
290 """PSF match, subtract, and decorrelate two images.
291
292 Parameters
293 ----------
294 template : `lsst.afw.image.ExposureF`
295 Template exposure, warped to match the science exposure.
296 science : `lsst.afw.image.ExposureF`
297 Science exposure to subtract from the template.
298 sources : `lsst.afw.table.SourceCatalog`
299 Identified sources on the science exposure. This catalog is used to
300 select sources in order to perform the AL PSF matching on stamp
301 images around them.
302 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
303 Exposure catalog with finalized psf models and aperture correction
304 maps to be applied. Catalog uses the detector id for the catalog
305 id, sorted on id for fast lookup. Deprecated in favor of
306 ``visitSummary``, and will be removed after v26.
307 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
308 Exposure catalog with external calibrations to be applied. Catalog
309 uses the detector id for the catalog id, sorted on id for fast
310 lookup. Ignored (for temporary backwards compatibility) if
311 ``finalizedPsfApCorrCatalog`` is provided.
312
313 Returns
314 -------
315 results : `lsst.pipe.base.Struct`
316 ``difference`` : `lsst.afw.image.ExposureF`
317 Result of subtracting template and science.
318 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
319 Warped and PSF-matched template exposure.
320 ``backgroundModel`` : `lsst.afw.math.Function2D`
321 Background model that was fit while solving for the
322 PSF-matching kernel
323 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
324 Kernel used to PSF-match the convolved image.
325
326 Raises
327 ------
328 RuntimeError
329 If an unsupported convolution mode is supplied.
330 RuntimeError
331 If there are too few sources to calculate the PSF matching kernel.
332 lsst.pipe.base.NoWorkFound
333 Raised if fraction of good pixels, defined as not having NO_DATA
334 set, is less then the configured requiredTemplateFraction
335 """
336
337 if finalizedPsfApCorrCatalog is not None:
338 warnings.warn(
339 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
340 "argument, and will be removed after v26.",
341 FutureWarning,
342 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
343 )
344 visitSummary = finalizedPsfApCorrCatalog
345
346 self._prepareInputs(template, science, visitSummary=visitSummary)
347
348 # In the event that getPsfFwhm fails, evaluate the PSF on a grid.
349 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
350 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
351
352 # Calling getPsfFwhm on template.psf fails on some rare occasions when
353 # the template has no input exposures at the average position of the
354 # stars. So we try getPsfFwhm first on template, and if that fails we
355 # evaluate the PSF on a grid specified by fwhmExposure* fields.
356 # To keep consistent definitions for PSF size on the template and
357 # science images, we use the same method for both.
358 try:
359 templatePsfSize = getPsfFwhm(template.psf)
360 sciencePsfSize = getPsfFwhm(science.psf)
361 except InvalidParameterError:
362 self.log.info("Unable to evaluate PSF at the average position. "
363 "Evaluting PSF on a grid of points."
364 )
365 templatePsfSize = evaluateMeanPsfFwhm(template,
366 fwhmExposureBuffer=fwhmExposureBuffer,
367 fwhmExposureGrid=fwhmExposureGrid
368 )
369 sciencePsfSize = evaluateMeanPsfFwhm(science,
370 fwhmExposureBuffer=fwhmExposureBuffer,
371 fwhmExposureGrid=fwhmExposureGrid
372 )
373 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize)
374 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize)
375 self.metadata.add("sciencePsfSize", sciencePsfSize)
376 self.metadata.add("templatePsfSize", templatePsfSize)
377 selectSources = self._sourceSelector(sources, science.mask)
378
379 if self.config.mode == "auto":
380 convolveTemplate = _shapeTest(template,
381 science,
382 fwhmExposureBuffer=fwhmExposureBuffer,
383 fwhmExposureGrid=fwhmExposureGrid)
384 if convolveTemplate:
385 if sciencePsfSize < templatePsfSize:
386 self.log.info("Average template PSF size is greater, "
387 "but science PSF greater in one dimension: convolving template image.")
388 else:
389 self.log.info("Science PSF size is greater: convolving template image.")
390 else:
391 self.log.info("Template PSF size is greater: convolving science image.")
392 elif self.config.mode == "convolveTemplate":
393 self.log.info("`convolveTemplate` is set: convolving template image.")
394 convolveTemplate = True
395 elif self.config.mode == "convolveScience":
396 self.log.info("`convolveScience` is set: convolving science image.")
397 convolveTemplate = False
398 else:
399 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
400
401 if convolveTemplate:
402 self.metadata.add("convolvedExposure", "Template")
403 subtractResults = self.runConvolveTemplate(template, science, selectSources)
404 else:
405 self.metadata.add("convolvedExposure", "Science")
406 subtractResults = self.runConvolveScience(template, science, selectSources)
407
408 return subtractResults
409
410 def runConvolveTemplate(self, template, science, selectSources):
411 """Convolve the template image with a PSF-matching kernel and subtract
412 from the science image.
413
414 Parameters
415 ----------
416 template : `lsst.afw.image.ExposureF`
417 Template exposure, warped to match the science exposure.
418 science : `lsst.afw.image.ExposureF`
419 Science exposure to subtract from the template.
420 selectSources : `lsst.afw.table.SourceCatalog`
421 Identified sources on the science exposure. This catalog is used to
422 select sources in order to perform the AL PSF matching on stamp
423 images around them.
424
425 Returns
426 -------
427 results : `lsst.pipe.base.Struct`
428
429 ``difference`` : `lsst.afw.image.ExposureF`
430 Result of subtracting template and science.
431 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
432 Warped and PSF-matched template exposure.
433 ``backgroundModel`` : `lsst.afw.math.Function2D`
434 Background model that was fit while solving for the PSF-matching kernel
435 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
436 Kernel used to PSF-match the template to the science image.
437 """
438 kernelSources = self.makeKernel.selectKernelSources(template, science,
439 candidateList=selectSources,
440 preconvolved=False)
441 kernelResult = self.makeKernel.run(template, science, kernelSources,
442 preconvolved=False)
443
444 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
446 bbox=science.getBBox(),
447 psf=science.psf,
448 photoCalib=science.photoCalib)
449
450 difference = _subtractImages(science, matchedTemplate,
451 backgroundModel=(kernelResult.backgroundModel
452 if self.config.doSubtractBackground else None))
453 correctedExposure = self.finalize(template, science, difference,
454 kernelResult.psfMatchingKernel,
455 templateMatched=True)
456
457 return lsst.pipe.base.Struct(difference=correctedExposure,
458 matchedTemplate=matchedTemplate,
459 matchedScience=science,
460 backgroundModel=kernelResult.backgroundModel,
461 psfMatchingKernel=kernelResult.psfMatchingKernel)
462
463 def runConvolveScience(self, template, science, selectSources):
464 """Convolve the science image with a PSF-matching kernel and subtract
465 the template image.
466
467 Parameters
468 ----------
469 template : `lsst.afw.image.ExposureF`
470 Template exposure, warped to match the science exposure.
471 science : `lsst.afw.image.ExposureF`
472 Science exposure to subtract from the template.
473 selectSources : `lsst.afw.table.SourceCatalog`
474 Identified sources on the science exposure. This catalog is used to
475 select sources in order to perform the AL PSF matching on stamp
476 images around them.
477
478 Returns
479 -------
480 results : `lsst.pipe.base.Struct`
481
482 ``difference`` : `lsst.afw.image.ExposureF`
483 Result of subtracting template and science.
484 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
485 Warped template exposure. Note that in this case, the template
486 is not PSF-matched to the science image.
487 ``backgroundModel`` : `lsst.afw.math.Function2D`
488 Background model that was fit while solving for the PSF-matching kernel
489 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
490 Kernel used to PSF-match the science image to the template.
491 """
492 bbox = science.getBBox()
493 kernelSources = self.makeKernel.selectKernelSources(science, template,
494 candidateList=selectSources,
495 preconvolved=False)
496 kernelResult = self.makeKernel.run(science, template, kernelSources,
497 preconvolved=False)
498 modelParams = kernelResult.backgroundModel.getParameters()
499 # We must invert the background model if the matching kernel is solved for the science image.
500 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
501
502 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
503 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False)
504
505 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel,
507 psf=template.psf)
508
509 # Place back on native photometric scale
510 matchedScience.maskedImage /= norm
511 matchedTemplate = template.clone()[bbox]
512 matchedTemplate.maskedImage /= norm
513 matchedTemplate.setPhotoCalib(science.photoCalib)
514
515 difference = _subtractImages(matchedScience, matchedTemplate,
516 backgroundModel=(kernelResult.backgroundModel
517 if self.config.doSubtractBackground else None))
518
519 correctedExposure = self.finalize(template, science, difference,
520 kernelResult.psfMatchingKernel,
521 templateMatched=False)
522
523 return lsst.pipe.base.Struct(difference=correctedExposure,
524 matchedTemplate=matchedTemplate,
525 matchedScience=matchedScience,
526 backgroundModel=kernelResult.backgroundModel,
527 psfMatchingKernel=kernelResult.psfMatchingKernel,)
528
529 def finalize(self, template, science, difference, kernel,
530 templateMatched=True,
531 preConvMode=False,
532 preConvKernel=None,
533 spatiallyVarying=False):
534 """Decorrelate the difference image to undo the noise correlations
535 caused by convolution.
536
537 Parameters
538 ----------
539 template : `lsst.afw.image.ExposureF`
540 Template exposure, warped to match the science exposure.
541 science : `lsst.afw.image.ExposureF`
542 Science exposure to subtract from the template.
543 difference : `lsst.afw.image.ExposureF`
544 Result of subtracting template and science.
545 kernel : `lsst.afw.math.Kernel`
546 An (optionally spatially-varying) PSF matching kernel
547 templateMatched : `bool`, optional
548 Was the template PSF-matched to the science image?
549 preConvMode : `bool`, optional
550 Was the science image preconvolved with its own PSF
551 before PSF matching the template?
552 preConvKernel : `lsst.afw.detection.Psf`, optional
553 If not `None`, then the science image was pre-convolved with
554 (the reflection of) this kernel. Must be normalized to sum to 1.
555 spatiallyVarying : `bool`, optional
556 Compute the decorrelation kernel spatially varying across the image?
557
558 Returns
559 -------
560 correctedExposure : `lsst.afw.image.ExposureF`
561 The decorrelated image difference.
562 """
563 # Erase existing detection mask planes.
564 # We don't want the detection mask from the science image
565
566 self.updateMasks(template, science, difference)
567
568 if self.config.doDecorrelation:
569 self.log.info("Decorrelating image difference.")
570 # We have cleared the template mask plane, so copy the mask plane of
571 # the image difference so that we can calculate correct statistics
572 # during decorrelation
573 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
574 templateMatched=templateMatched,
575 preConvMode=preConvMode,
576 preConvKernel=preConvKernel,
577 spatiallyVarying=spatiallyVarying).correctedExposure
578 else:
579 self.log.info("NOT decorrelating image difference.")
580 correctedExposure = difference
581 return correctedExposure
582
583 def updateMasks(self, template, science, difference):
584 """Update the mask planes on images for finalizing."""
585
586 bbox = science.getBBox()
587 mask = difference.mask
588 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
589
590 if "FAKE" in science.mask.getMaskPlaneDict().keys():
591 # propagate the mask plane related to Fake source injection
592 # NOTE: the fake source injection sets FAKE plane, but it should be INJECTED
593 # NOTE: This can be removed in DM-40796
594
595 self.log.info("Adding injected mask planes")
596 mask.addMaskPlane("INJECTED")
597 diffInjectedBitMask = mask.getPlaneBitMask("INJECTED")
598
599 mask.addMaskPlane("INJECTED_TEMPLATE")
600 diffInjTmpltBitMask = mask.getPlaneBitMask("INJECTED_TEMPLATE")
601
602 scienceFakeBitMask = science.mask.getPlaneBitMask('FAKE')
603 tmpltFakeBitMask = template[bbox].mask.getPlaneBitMask('FAKE')
604
605 injScienceMaskArray = ((science.mask.array & scienceFakeBitMask) > 0) * diffInjectedBitMask
606 injTemplateMaskArray = ((template[bbox].mask.array & tmpltFakeBitMask) > 0) * diffInjTmpltBitMask
607
608 mask.array |= injScienceMaskArray
609 mask.array |= injTemplateMaskArray
610
611 template[bbox].mask.array[...] = difference.mask.array[...]
612
613 @staticmethod
614 def _validateExposures(template, science):
615 """Check that the WCS of the two Exposures match, and the template bbox
616 contains the science bbox.
617
618 Parameters
619 ----------
620 template : `lsst.afw.image.ExposureF`
621 Template exposure, warped to match the science exposure.
622 science : `lsst.afw.image.ExposureF`
623 Science exposure to subtract from the template.
624
625 Raises
626 ------
627 AssertionError
628 Raised if the WCS of the template is not equal to the science WCS,
629 or if the science image is not fully contained in the template
630 bounding box.
631 """
632 assert template.wcs == science.wcs,\
633 "Template and science exposure WCS are not identical."
634 templateBBox = template.getBBox()
635 scienceBBox = science.getBBox()
636
637 assert templateBBox.contains(scienceBBox),\
638 "Template bbox does not contain all of the science image."
639
640 def _convolveExposure(self, exposure, kernel, convolutionControl,
641 bbox=None,
642 psf=None,
643 photoCalib=None,
644 interpolateBadMaskPlanes=False,
645 ):
646 """Convolve an exposure with the given kernel.
647
648 Parameters
649 ----------
650 exposure : `lsst.afw.Exposure`
651 exposure to convolve.
652 kernel : `lsst.afw.math.LinearCombinationKernel`
653 PSF matching kernel computed in the ``makeKernel`` subtask.
654 convolutionControl : `lsst.afw.math.ConvolutionControl`
655 Configuration for convolve algorithm.
656 bbox : `lsst.geom.Box2I`, optional
657 Bounding box to trim the convolved exposure to.
658 psf : `lsst.afw.detection.Psf`, optional
659 Point spread function (PSF) to set for the convolved exposure.
660 photoCalib : `lsst.afw.image.PhotoCalib`, optional
661 Photometric calibration of the convolved exposure.
662
663 Returns
664 -------
665 convolvedExp : `lsst.afw.Exposure`
666 The convolved image.
667 """
668 convolvedExposure = exposure.clone()
669 if psf is not None:
670 convolvedExposure.setPsf(psf)
671 if photoCalib is not None:
672 convolvedExposure.setPhotoCalib(photoCalib)
673 if interpolateBadMaskPlanes and self.config.badMaskPlanes is not None:
674 nInterp = _interpolateImage(convolvedExposure.maskedImage,
675 self.config.badMaskPlanes)
676 self.metadata.add("nInterpolated", nInterp)
677 convolvedImage = lsst.afw.image.MaskedImageF(convolvedExposure.getBBox())
678 lsst.afw.math.convolve(convolvedImage, convolvedExposure.maskedImage, kernel, convolutionControl)
679 convolvedExposure.setMaskedImage(convolvedImage)
680 if bbox is None:
681 return convolvedExposure
682 else:
683 return convolvedExposure[bbox]
684
685 def _sourceSelector(self, sources, mask):
686 """Select sources from a catalog that meet the selection criteria.
687
688 Parameters
689 ----------
690 sources : `lsst.afw.table.SourceCatalog`
691 Input source catalog to select sources from.
692 mask : `lsst.afw.image.Mask`
693 The image mask plane to use to reject sources
694 based on their location on the ccd.
695
696 Returns
697 -------
698 selectSources : `lsst.afw.table.SourceCatalog`
699 The input source catalog, with flagged and low signal-to-noise
700 sources removed.
701
702 Raises
703 ------
704 RuntimeError
705 If there are too few sources to compute the PSF matching kernel
706 remaining after source selection.
707 """
708 flags = np.ones(len(sources), dtype=bool)
709 for flag in self.config.badSourceFlags:
710 try:
711 flags *= ~sources[flag]
712 except Exception as e:
713 self.log.warning("Could not apply source flag: %s", e)
714 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
715 flags *= sToNFlag
716 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes)
717 selectSources = sources[flags]
718 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog",
719 len(selectSources), len(sources), 100*len(selectSources)/len(sources))
720 if len(selectSources) < self.config.makeKernel.nStarPerCell:
721 self.log.error("Too few sources to calculate the PSF matching kernel: "
722 "%i selected but %i needed for the calculation.",
723 len(selectSources), self.config.makeKernel.nStarPerCell)
724 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.")
725 self.metadata.add("nPsfSources", len(selectSources))
726
727 return selectSources.copy(deep=True)
728
729 @staticmethod
730 def _checkMask(mask, sources, badMaskPlanes):
731 """Exclude sources that are located on masked pixels.
732
733 Parameters
734 ----------
735 mask : `lsst.afw.image.Mask`
736 The image mask plane to use to reject sources
737 based on the location of their centroid on the ccd.
738 sources : `lsst.afw.table.SourceCatalog`
739 The source catalog to evaluate.
740 badMaskPlanes : `list` of `str`
741 List of the names of the mask planes to exclude.
742
743 Returns
744 -------
745 flags : `numpy.ndarray` of `bool`
746 Array indicating whether each source in the catalog should be
747 kept (True) or rejected (False) based on the value of the
748 mask plane at its location.
749 """
750 setBadMaskPlanes = [
751 maskPlane for maskPlane in badMaskPlanes if maskPlane in mask.getMaskPlaneDict()
752 ]
753
754 badPixelMask = mask.getPlaneBitMask(setBadMaskPlanes)
755
756 xv = np.rint(sources.getX() - mask.getX0())
757 yv = np.rint(sources.getY() - mask.getY0())
758
759 mv = mask.array[yv.astype(int), xv.astype(int)]
760 flags = np.bitwise_and(mv, badPixelMask) == 0
761 return flags
762
763 def _prepareInputs(self, template, science, visitSummary=None):
764 """Perform preparatory calculations common to all Alard&Lupton Tasks.
765
766 Parameters
767 ----------
768 template : `lsst.afw.image.ExposureF`
769 Template exposure, warped to match the science exposure. The
770 variance plane of the template image is modified in place.
771 science : `lsst.afw.image.ExposureF`
772 Science exposure to subtract from the template. The variance plane
773 of the science image is modified in place.
774 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
775 Exposure catalog with external calibrations to be applied. Catalog
776 uses the detector id for the catalog id, sorted on id for fast
777 lookup.
778 """
779 self._validateExposures(template, science)
780 if visitSummary is not None:
781 self._applyExternalCalibrations(science, visitSummary=visitSummary)
783 requiredTemplateFraction=self.config.requiredTemplateFraction)
784
785 if self.config.doScaleVariance:
786 # Scale the variance of the template and science images before
787 # convolution, subtraction, or decorrelation so that they have the
788 # correct ratio.
789 templateVarFactor = self.scaleVariance.run(template.maskedImage)
790 sciVarFactor = self.scaleVariance.run(science.maskedImage)
791 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
792 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
793 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
794 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor)
795 self._clearMask(template)
796
797 def _clearMask(self, template):
798 """Clear the mask plane of the template.
799
800 Parameters
801 ----------
802 template : `lsst.afw.image.ExposureF`
803 Template exposure, warped to match the science exposure.
804 The mask plane will be modified in place.
805 """
806 mask = template.mask
807 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys()
808 if maskplane not in self.config.preserveTemplateMask]
809
810 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes)
811 mask &= ~bitMaskToClear
812
813
815 SubtractScoreOutputConnections):
816 pass
817
818
820 pipelineConnections=AlardLuptonPreconvolveSubtractConnections):
821 pass
822
823
825 """Subtract a template from a science image, convolving the science image
826 before computing the kernel, and also convolving the template before
827 subtraction.
828 """
829 ConfigClass = AlardLuptonPreconvolveSubtractConfig
830 _DefaultName = "alardLuptonPreconvolveSubtract"
831
832 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None):
833 """Preconvolve the science image with its own PSF,
834 convolve the template image with a PSF-matching kernel and subtract
835 from the preconvolved science image.
836
837 Parameters
838 ----------
839 template : `lsst.afw.image.ExposureF`
840 The template image, which has previously been warped to the science
841 image. The template bbox will be padded by a few pixels compared to
842 the science bbox.
843 science : `lsst.afw.image.ExposureF`
844 The science exposure.
845 sources : `lsst.afw.table.SourceCatalog`
846 Identified sources on the science exposure. This catalog is used to
847 select sources in order to perform the AL PSF matching on stamp
848 images around them.
849 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
850 Exposure catalog with finalized psf models and aperture correction
851 maps to be applied. Catalog uses the detector id for the catalog
852 id, sorted on id for fast lookup. Deprecated in favor of
853 ``visitSummary``, and will be removed after v26.
854 visitSummary : `lsst.afw.table.ExposureCatalog`, optional
855 Exposure catalog with complete external calibrations. Catalog uses
856 the detector id for the catalog id, sorted on id for fast lookup.
857 Ignored (for temporary backwards compatibility) if
858 ``finalizedPsfApCorrCatalog`` is provided.
859
860 Returns
861 -------
862 results : `lsst.pipe.base.Struct`
863 ``scoreExposure`` : `lsst.afw.image.ExposureF`
864 Result of subtracting the convolved template and science
865 images. Attached PSF is that of the original science image.
866 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
867 Warped and PSF-matched template exposure. Attached PSF is that
868 of the original science image.
869 ``matchedScience`` : `lsst.afw.image.ExposureF`
870 The science exposure after convolving with its own PSF.
871 Attached PSF is that of the original science image.
872 ``backgroundModel`` : `lsst.afw.math.Function2D`
873 Background model that was fit while solving for the
874 PSF-matching kernel
875 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
876 Final kernel used to PSF-match the template to the science
877 image.
878 """
879 if finalizedPsfApCorrCatalog is not None:
880 warnings.warn(
881 "The finalizedPsfApCorrCatalog argument is deprecated in favor of the visitSummary "
882 "argument, and will be removed after v26.",
883 FutureWarning,
884 stacklevel=find_outside_stacklevel("lsst.ip.diffim"),
885 )
886 visitSummary = finalizedPsfApCorrCatalog
887
888 self._prepareInputs(template, science, visitSummary=visitSummary)
889
890 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation
891 scienceKernel = science.psf.getKernel()
892 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControlconvolutionControl,
893 interpolateBadMaskPlanes=True)
894 selectSources = self._sourceSelector(sources, matchedScience.mask)
895 self.metadata.add("convolvedExposure", "Preconvolution")
896
897 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel)
898
899 return subtractResults
900
901 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel):
902 """Convolve the science image with its own PSF, then convolve the
903 template with a matching kernel and subtract to form the Score
904 exposure.
905
906 Parameters
907 ----------
908 template : `lsst.afw.image.ExposureF`
909 Template exposure, warped to match the science exposure.
910 science : `lsst.afw.image.ExposureF`
911 Science exposure to subtract from the template.
912 matchedScience : `lsst.afw.image.ExposureF`
913 The science exposure, convolved with the reflection of its own PSF.
914 selectSources : `lsst.afw.table.SourceCatalog`
915 Identified sources on the science exposure. This catalog is used to
916 select sources in order to perform the AL PSF matching on stamp
917 images around them.
918 preConvKernel : `lsst.afw.math.Kernel`
919 The reflection of the kernel that was used to preconvolve the
920 `science` exposure. Must be normalized to sum to 1.
921
922 Returns
923 -------
924 results : `lsst.pipe.base.Struct`
925
926 ``scoreExposure`` : `lsst.afw.image.ExposureF`
927 Result of subtracting the convolved template and science
928 images. Attached PSF is that of the original science image.
929 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
930 Warped and PSF-matched template exposure. Attached PSF is that
931 of the original science image.
932 ``matchedScience`` : `lsst.afw.image.ExposureF`
933 The science exposure after convolving with its own PSF.
934 Attached PSF is that of the original science image.
935 ``backgroundModel`` : `lsst.afw.math.Function2D`
936 Background model that was fit while solving for the
937 PSF-matching kernel
938 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
939 Final kernel used to PSF-match the template to the science
940 image.
941 """
942 bbox = science.getBBox()
943 innerBBox = preConvKernel.shrinkBBox(bbox)
944
945 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox],
946 candidateList=selectSources,
947 preconvolved=True)
948 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources,
949 preconvolved=True)
950
951 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
953 bbox=bbox,
954 psf=science.psf,
955 interpolateBadMaskPlanes=True,
956 photoCalib=science.photoCalib)
957 score = _subtractImages(matchedScience, matchedTemplate,
958 backgroundModel=(kernelResult.backgroundModel
959 if self.config.doSubtractBackground else None))
960 correctedScore = self.finalize(template[bbox], science, score,
961 kernelResult.psfMatchingKernel,
962 templateMatched=True, preConvMode=True,
963 preConvKernel=preConvKernel)
964
965 return lsst.pipe.base.Struct(scoreExposure=correctedScore,
966 matchedTemplate=matchedTemplate,
967 matchedScience=matchedScience,
968 backgroundModel=kernelResult.backgroundModel,
969 psfMatchingKernel=kernelResult.psfMatchingKernel)
970
971
972def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.):
973 """Raise NoWorkFound if template coverage < requiredTemplateFraction
974
975 Parameters
976 ----------
977 templateExposure : `lsst.afw.image.ExposureF`
978 The template exposure to check
979 logger : `lsst.log.Log`
980 Logger for printing output.
981 requiredTemplateFraction : `float`, optional
982 Fraction of pixels of the science image required to have coverage
983 in the template.
984
985 Raises
986 ------
987 lsst.pipe.base.NoWorkFound
988 Raised if fraction of good pixels, defined as not having NO_DATA
989 set, is less then the configured requiredTemplateFraction
990 """
991 # Count the number of pixels with the NO_DATA mask bit set
992 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
993 pixNoData = np.count_nonzero(templateExposure.mask.array
994 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
995 pixGood = templateExposure.getBBox().getArea() - pixNoData
996 logger.info("template has %d good pixels (%.1f%%)", pixGood,
997 100*pixGood/templateExposure.getBBox().getArea())
998
999 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
1000 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
1001 "To force subtraction, set config requiredTemplateFraction=0." % (
1002 100*pixGood/templateExposure.getBBox().getArea(),
1003 100*requiredTemplateFraction))
1004 raise lsst.pipe.base.NoWorkFound(message)
1005
1006
1007def _subtractImages(science, template, backgroundModel=None):
1008 """Subtract template from science, propagating relevant metadata.
1009
1010 Parameters
1011 ----------
1012 science : `lsst.afw.Exposure`
1013 The input science image.
1014 template : `lsst.afw.Exposure`
1015 The template to subtract from the science image.
1016 backgroundModel : `lsst.afw.MaskedImage`, optional
1017 Differential background model
1018
1019 Returns
1020 -------
1021 difference : `lsst.afw.Exposure`
1022 The subtracted image.
1023 """
1024 difference = science.clone()
1025 if backgroundModel is not None:
1026 difference.maskedImage -= backgroundModel
1027 difference.maskedImage -= template.maskedImage
1028 return difference
1029
1030
1031def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
1032 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
1033
1034 Parameters
1035 ----------
1036 exp1 : `~lsst.afw.image.Exposure`
1037 Exposure with the reference point spread function (PSF) to evaluate.
1038 exp2 : `~lsst.afw.image.Exposure`
1039 Exposure with a candidate point spread function (PSF) to evaluate.
1040 fwhmExposureBuffer : `float`
1041 Fractional buffer margin to be left out of all sides of the image
1042 during the construction of the grid to compute mean PSF FWHM in an
1043 exposure, if the PSF is not available at its average position.
1044 fwhmExposureGrid : `int`
1045 Grid size to compute the mean FWHM in an exposure, if the PSF is not
1046 available at its average position.
1047 Returns
1048 -------
1049 result : `bool`
1050 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in
1051 either dimension.
1052 """
1053 try:
1054 shape1 = getPsfFwhm(exp1.psf, average=False)
1055 shape2 = getPsfFwhm(exp2.psf, average=False)
1056 except InvalidParameterError:
1057 shape1 = evaluateMeanPsfFwhm(exp1,
1058 fwhmExposureBuffer=fwhmExposureBuffer,
1059 fwhmExposureGrid=fwhmExposureGrid
1060 )
1061 shape2 = evaluateMeanPsfFwhm(exp2,
1062 fwhmExposureBuffer=fwhmExposureBuffer,
1063 fwhmExposureGrid=fwhmExposureGrid
1064 )
1065 return shape1 <= shape2
1066
1067 # Results from getPsfFwhm is a tuple of two values, one for each dimension.
1068 xTest = shape1[0] <= shape2[0]
1069 yTest = shape1[1] <= shape2[1]
1070 return xTest | yTest
1071
1072
1073def _interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None):
1074 """Replace masked image pixels with interpolated values.
1075
1076 Parameters
1077 ----------
1078 maskedImage : `lsst.afw.image.MaskedImage`
1079 Image on which to perform interpolation.
1080 badMaskPlanes : `list` of `str`
1081 List of mask planes to interpolate over.
1082 fallbackValue : `float`, optional
1083 Value to set when interpolation fails.
1084
1085 Returns
1086 -------
1087 result: `float`
1088 The number of masked pixels that were replaced.
1089 """
1090 imgBadMaskPlanes = [
1091 maskPlane for maskPlane in badMaskPlanes if maskPlane in maskedImage.mask.getMaskPlaneDict()
1092 ]
1093
1094 image = maskedImage.image.array
1095 badPixels = (maskedImage.mask.array & maskedImage.mask.getPlaneBitMask(imgBadMaskPlanes)) > 0
1096 image[badPixels] = np.nan
1097 if fallbackValue is None:
1098 fallbackValue = np.nanmedian(image)
1099 # For this initial implementation, skip the interpolation and just fill with
1100 # the median value.
1101 image[badPixels] = fallbackValue
1102 return np.sum(badPixels)
Asseses the quality of a candidate given a spatial kernel and background model.
runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel)
run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None)
_prepareInputs(self, template, science, visitSummary=None)
runConvolveTemplate(self, template, science, selectSources)
_convolveExposure(self, exposure, kernel, convolutionControl, bbox=None, psf=None, photoCalib=None, interpolateBadMaskPlanes=False)
runConvolveScience(self, template, science, selectSources)
updateMasks(self, template, science, difference)
run(self, template, science, sources, finalizedPsfApCorrCatalog=None, visitSummary=None)
finalize(self, template, science, difference, kernel, templateMatched=True, preConvMode=False, preConvKernel=None, spatiallyVarying=False)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, ConvolutionControl const &convolutionControl=ConvolutionControl())
_subtractImages(science, template, backgroundModel=None)
_interpolateImage(maskedImage, badMaskPlanes, fallbackValue=None)
checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.)
_shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid)