lsst.ip.diffim gf5af78f4f3+36b4fd9b9f
subtractImages.py
Go to the documentation of this file.
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import numpy as np
23
24import lsst.afw.image
25import lsst.afw.math
26import lsst.geom
27from lsst.ip.diffim.utils import getPsfFwhm
28from lsst.meas.algorithms import ScaleVarianceTask
29import lsst.pex.config
30import lsst.pipe.base
31from lsst.pipe.base import connectionTypes
32from . import MakeKernelTask, DecorrelateALKernelTask
33
34__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask"]
35
36_dimensions = ("instrument", "visit", "detector")
37_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
38
39
40class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
41 dimensions=_dimensions,
42 defaultTemplates=_defaultTemplates):
43 template = connectionTypes.Input(
44 doc="Input warped template to subtract.",
45 dimensions=("instrument", "visit", "detector"),
46 storageClass="ExposureF",
47 name="{fakesType}{coaddName}Diff_templateExp"
48 )
49 science = connectionTypes.Input(
50 doc="Input science exposure to subtract from.",
51 dimensions=("instrument", "visit", "detector"),
52 storageClass="ExposureF",
53 name="{fakesType}calexp"
54 )
55 sources = connectionTypes.Input(
56 doc="Sources measured on the science exposure; "
57 "used to select sources for making the matching kernel.",
58 dimensions=("instrument", "visit", "detector"),
59 storageClass="SourceCatalog",
60 name="{fakesType}src"
61 )
62
63
64class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
65 dimensions=_dimensions,
66 defaultTemplates=_defaultTemplates):
67 difference = connectionTypes.Output(
68 doc="Result of subtracting convolved template from science image.",
69 dimensions=("instrument", "visit", "detector"),
70 storageClass="ExposureF",
71 name="{fakesType}{coaddName}Diff_differenceTempExp",
72 )
73 matchedTemplate = connectionTypes.Output(
74 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
75 dimensions=("instrument", "visit", "detector"),
76 storageClass="ExposureF",
77 name="{fakesType}{coaddName}Diff_matchedExp",
78 )
79
80
82 pass
83
84
85class AlardLuptonSubtractConfig(lsst.pipe.base.PipelineTaskConfig,
86 pipelineConnections=AlardLuptonSubtractConnections):
87 mode = lsst.pex.config.ChoiceField(
88 dtype=str,
89 default="auto",
90 allowed={"auto": "Choose which image to convolve at runtime.",
91 "convolveScience": "Only convolve the science image.",
92 "convolveTemplate": "Only convolve the template image."},
93 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
94 )
95 makeKernel = lsst.pex.config.ConfigurableField(
96 target=MakeKernelTask,
97 doc="Task to construct a matching kernel for convolution.",
98 )
99 doDecorrelation = lsst.pex.config.Field(
100 dtype=bool,
101 default=True,
102 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
103 "kernel convolution? If True, also update the diffim PSF."
104 )
105 decorrelate = lsst.pex.config.ConfigurableField(
106 target=DecorrelateALKernelTask,
107 doc="Task to decorrelate the image difference.",
108 )
109 requiredTemplateFraction = lsst.pex.config.Field(
110 dtype=float,
111 default=0.1,
112 doc="Abort task if template covers less than this fraction of pixels."
113 " Setting to 0 will always attempt image subtraction."
114 )
115 doScaleVariance = lsst.pex.config.Field(
116 dtype=bool,
117 default=True,
118 doc="Scale variance of the image difference?"
119 )
120 scaleVariance = lsst.pex.config.ConfigurableField(
121 target=ScaleVarianceTask,
122 doc="Subtask to rescale the variance of the template to the statistically expected level."
123 )
124 doSubtractBackground = lsst.pex.config.Field(
125 doc="Subtract the background fit when solving the kernel?",
126 dtype=bool,
127 default=True,
128 )
129
130 forceCompatibility = lsst.pex.config.Field(
131 dtype=bool,
132 default=False,
133 doc="Set up and run diffim using settings that ensure the results"
134 "are compatible with the old version in pipe_tasks.",
135 deprecated="This option is only for backwards compatibility purposes"
136 " and will be removed after v24.",
137 )
138
139 def setDefaults(self):
140 self.makeKernelmakeKernel.kernel.name = "AL"
141 self.makeKernelmakeKernel.kernel.active.fitForBackground = self.doSubtractBackgrounddoSubtractBackground
142 self.makeKernelmakeKernel.kernel.active.spatialKernelOrder = 1
143 self.makeKernelmakeKernel.kernel.active.spatialBgOrder = 2
144 if self.forceCompatibilityforceCompatibility:
145 self.modemode = "convolveTemplate"
146
147
148class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
149 """Compute the image difference of a science and template image using
150 the Alard & Lupton (1998) algorithm.
151 """
152 ConfigClass = AlardLuptonSubtractConfig
153 _DefaultName = "alardLuptonSubtract"
154
155 def __init__(self, **kwargs):
156 super().__init__(**kwargs)
157 self.makeSubtask("decorrelate")
158 self.makeSubtask("makeKernel")
159 if self.config.doScaleVariance:
160 self.makeSubtask("scaleVariance")
161
163 # Normalization is an extra, unnecessary, calculation and will result
164 # in mis-subtraction of the images if there are calibration errors.
165 self.convolutionControlconvolutionControl.setDoNormalize(False)
166
167 def run(self, template, science, sources):
168 """PSF match, subtract, and decorrelate two images.
169
170 Parameters
171 ----------
172 template : `lsst.afw.image.ExposureF`
173 Template exposure, warped to match the science exposure.
174 science : `lsst.afw.image.ExposureF`
175 Science exposure to subtract from the template.
177 Identified sources on the science exposure. This catalog is used to
178 select sources in order to perform the AL PSF matching on stamp
179 images around them.
180
181 Returns
182 -------
183 results : `lsst.pipe.base.Struct`
184 ``difference`` : `lsst.afw.image.ExposureF`
185 Result of subtracting template and science.
186 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
187 Warped and PSF-matched template exposure.
188 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D`
189 Background model that was fit while solving for the PSF-matching kernel
190 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
191 Kernel used to PSF-match the convolved image.
192
193 Raises
194 ------
195 RuntimeError
196 If an unsupported convolution mode is supplied.
197 lsst.pipe.base.NoWorkFound
198 Raised if fraction of good pixels, defined as not having NO_DATA
199 set, is less then the configured requiredTemplateFraction
200 """
201 self._validateExposures_validateExposures(template, science)
202 checkTemplateIsSufficient(template, self.log,
203 requiredTemplateFraction=self.config.requiredTemplateFraction)
204 if self.config.forceCompatibility:
205 # Compatibility option to maintain old functionality
206 # This should be removed in the future!
207 sources = None
208 kernelSources = self.makeKernel.selectKernelSources(template, science,
209 candidateList=sources,
210 preconvolved=False)
211 sciencePsfSize = getPsfFwhm(science.psf)
212 templatePsfSize = getPsfFwhm(template.psf)
213 self.log.info("Science PSF size: %f", sciencePsfSize)
214 self.log.info("Template PSF size: %f", templatePsfSize)
215 if self.config.mode == "auto":
216 if sciencePsfSize < templatePsfSize:
217 self.log.info("Template PSF size is greater: convolving science image.")
218 convolveTemplate = False
219 else:
220 self.log.info("Science PSF size is greater: convolving template image.")
221 convolveTemplate = True
222 elif self.config.mode == "convolveTemplate":
223 self.log.info("`convolveTemplate` is set: convolving template image.")
224 convolveTemplate = True
225 elif self.config.mode == "convolveScience":
226 self.log.info("`convolveScience` is set: convolving science image.")
227 convolveTemplate = False
228 else:
229 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
230
231 if convolveTemplate:
232 subtractResults = self.runConvolveTemplaterunConvolveTemplate(template, science, kernelSources)
233 else:
234 subtractResults = self.runConvolveSciencerunConvolveScience(template, science, kernelSources)
235
236 if self.config.doScaleVariance:
237 diffimVarFactor = self.scaleVariance.run(subtractResults.difference.maskedImage)
238 self.log.info("Diffim variance scaling factor: %.2f", diffimVarFactor)
239 self.metadata.add("scaleDiffimVarianceFactor", diffimVarFactor)
240
241 return subtractResults
242
243 def runConvolveTemplate(self, template, science, sources):
244 """Convolve the template image with a PSF-matching kernel and subtract
245 from the science image.
246
247 Parameters
248 ----------
249 template : `lsst.afw.image.ExposureF`
250 Template exposure, warped to match the science exposure.
251 science : `lsst.afw.image.ExposureF`
252 Science exposure to subtract from the template.
254 Identified sources on the science exposure. This catalog is used to
255 select sources in order to perform the AL PSF matching on stamp
256 images around them.
257
258 Returns
259 -------
260 results : `lsst.pipe.base.Struct`
261
262 ``difference`` : `lsst.afw.image.ExposureF`
263 Result of subtracting template and science.
264 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
265 Warped and PSF-matched template exposure.
266 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D`
267 Background model that was fit while solving for the PSF-matching kernel
268 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
269 Kernel used to PSF-match the template to the science image.
270 """
271 if self.config.forceCompatibility:
272 # Compatibility option to maintain old behavior
273 # This should be removed in the future!
274 template = template[science.getBBox()]
275 kernelResult = self.makeKernel.run(template, science, sources, preconvolved=False)
276
277 matchedTemplate = self._convolveExposure_convolveExposure(template, kernelResult.psfMatchingKernel,
278 self.convolutionControlconvolutionControl,
279 bbox=science.getBBox(),
280 psf=science.psf)
281 difference = _subtractImages(science, matchedTemplate,
282 backgroundModel=(kernelResult.backgroundModel
283 if self.config.doSubtractBackground else None))
284 correctedExposure = self.finalizefinalize(template, science, difference, kernelResult.psfMatchingKernel,
285 templateMatched=True)
286
287 return lsst.pipe.base.Struct(difference=correctedExposure,
288 matchedTemplate=matchedTemplate,
289 backgroundModel=kernelResult.backgroundModel,
290 psfMatchingKernel=kernelResult.psfMatchingKernel)
291
292 def runConvolveScience(self, template, science, sources):
293 """Convolve the science image with a PSF-matching kernel and subtract the template image.
294
295 Parameters
296 ----------
297 template : `lsst.afw.image.ExposureF`
298 Template exposure, warped to match the science exposure.
299 science : `lsst.afw.image.ExposureF`
300 Science exposure to subtract from the template.
302 Identified sources on the science exposure. This catalog is used to
303 select sources in order to perform the AL PSF matching on stamp
304 images around them.
305
306 Returns
307 -------
308 results : `lsst.pipe.base.Struct`
309
310 ``difference`` : `lsst.afw.image.ExposureF`
311 Result of subtracting template and science.
312 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
313 Warped template exposure. Note that in this case, the template
314 is not PSF-matched to the science image.
315 ``backgroundModel`` : `lsst.afw.math.Chebyshev1Function2D`
316 Background model that was fit while solving for the PSF-matching kernel
317 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
318 Kernel used to PSF-match the science image to the template.
319 """
320 if self.config.forceCompatibility:
321 # Compatibility option to maintain old behavior
322 # This should be removed in the future!
323 template = template[science.getBBox()]
324 kernelResult = self.makeKernel.run(science, template, sources, preconvolved=False)
325 modelParams = kernelResult.backgroundModel.getParameters()
326 # We must invert the background model if the matching kernel is solved for the science image.
327 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
328
329 matchedScience = self._convolveExposure_convolveExposure(science, kernelResult.psfMatchingKernel,
330 self.convolutionControlconvolutionControl,
331 psf=template.psf)
332
333 difference = _subtractImages(matchedScience, template[science.getBBox()],
334 backgroundModel=(kernelResult.backgroundModel
335 if self.config.doSubtractBackground else None))
336
337 # Place back on native photometric scale
338 difference.maskedImage /= kernelResult.psfMatchingKernel.computeImage(
339 lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions()), False)
340 correctedExposure = self.finalizefinalize(template, science, difference, kernelResult.psfMatchingKernel,
341 templateMatched=False)
342
343 return lsst.pipe.base.Struct(difference=correctedExposure,
344 matchedTemplate=template,
345 backgroundModel=kernelResult.backgroundModel,
346 psfMatchingKernel=kernelResult.psfMatchingKernel,)
347
348 def finalize(self, template, science, difference, kernel,
349 templateMatched=True,
350 preConvMode=False,
351 preConvKernel=None,
352 spatiallyVarying=False):
353 """Decorrelate the difference image to undo the noise correlations
354 caused by convolution.
355
356 Parameters
357 ----------
358 template : `lsst.afw.image.ExposureF`
359 Template exposure, warped to match the science exposure.
360 science : `lsst.afw.image.ExposureF`
361 Science exposure to subtract from the template.
362 difference : `lsst.afw.image.ExposureF`
363 Result of subtracting template and science.
364 kernel : `lsst.afw.math.Kernel`
365 An (optionally spatially-varying) PSF matching kernel
366 templateMatched : `bool`, optional
367 Was the template PSF-matched to the science image?
368 preConvMode : `bool`, optional
369 Was the science image preconvolved with its own PSF
370 before PSF matching the template?
371 preConvKernel : `lsst.afw.detection.Psf`, optional
372 If not `None`, then the science image was pre-convolved with
373 (the reflection of) this kernel. Must be normalized to sum to 1.
374 spatiallyVarying : `bool`, optional
375 Compute the decorrelation kernel spatially varying across the image?
376
377 Returns
378 -------
379 correctedExposure : `lsst.afw.image.ExposureF`
380 The decorrelated image difference.
381 """
382 # Erase existing detection mask planes.
383 # We don't want the detection mask from the science image
384 mask = difference.mask
385 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
386
387 if self.config.doDecorrelation:
388 self.log.info("Decorrelating image difference.")
389 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
390 templateMatched=templateMatched,
391 preConvMode=preConvMode,
392 preConvKernel=preConvKernel,
393 spatiallyVarying=spatiallyVarying).correctedExposure
394 else:
395 self.log.info("NOT decorrelating image difference.")
396 correctedExposure = difference
397 return correctedExposure
398
399 @staticmethod
400 def _validateExposures(template, science):
401 """Check that the WCS of the two Exposures match, and the template bbox
402 contains the science bbox.
403
404 Parameters
405 ----------
406 template : `lsst.afw.image.ExposureF`
407 Template exposure, warped to match the science exposure.
408 science : `lsst.afw.image.ExposureF`
409 Science exposure to subtract from the template.
410
411 Raises
412 ------
413 AssertionError
414 Raised if the WCS of the template is not equal to the science WCS,
415 or if the science image is not fully contained in the template
416 bounding box.
417 """
418 assert template.wcs == science.wcs,\
419 "Template and science exposure WCS are not identical."
420 templateBBox = template.getBBox()
421 scienceBBox = science.getBBox()
422
423 assert templateBBox.contains(scienceBBox),\
424 "Template bbox does not contain all of the science image."
425
426 @staticmethod
427 def _convolveExposure(exposure, kernel, convolutionControl,
428 bbox=None,
429 psf=None):
430 """Convolve an exposure with the given kernel.
431
432 Parameters
433 ----------
434 exposure : `lsst.afw.Exposure`
435 exposure to convolve.
437 PSF matching kernel computed in the ``makeKernel`` subtask.
438 convolutionControl : `lsst.afw.math.ConvolutionControl`
439 Configuration for convolve algorithm.
440 bbox : `lsst.geom.Box2I`, optional
441 Bounding box to trim the convolved exposure to.
442 psf : `lsst.afw.detection.Psf`, optional
443 Point spread function (PSF) to set for the convolved exposure.
444
445 Returns
446 -------
447 convolvedExp : `lsst.afw.Exposure`
448 The convolved image.
449 """
450 convolvedExposure = exposure.clone()
451 if psf is not None:
452 convolvedExposure.setPsf(psf)
453 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox())
454 lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl)
455 convolvedExposure.setMaskedImage(convolvedImage)
456 if bbox is None:
457 return convolvedExposure
458 else:
459 return convolvedExposure[bbox]
460
461
462def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.):
463 """Raise NoWorkFound if template coverage < requiredTemplateFraction
464
465 Parameters
466 ----------
467 templateExposure : `lsst.afw.image.ExposureF`
468 The template exposure to check
469 logger : `lsst.log.Log`
470 Logger for printing output.
471 requiredTemplateFraction : `float`, optional
472 Fraction of pixels of the science image required to have coverage
473 in the template.
474
475 Raises
476 ------
477 lsst.pipe.base.NoWorkFound
478 Raised if fraction of good pixels, defined as not having NO_DATA
479 set, is less then the configured requiredTemplateFraction
480 """
481 # Count the number of pixels with the NO_DATA mask bit set
482 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
483 pixNoData = np.count_nonzero(templateExposure.mask.array
484 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
485 pixGood = templateExposure.getBBox().getArea() - pixNoData
486 logger.info("template has %d good pixels (%.1f%%)", pixGood,
487 100*pixGood/templateExposure.getBBox().getArea())
488
489 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
490 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
491 "To force subtraction, set config requiredTemplateFraction=0." % (
492 100*pixGood/templateExposure.getBBox().getArea(),
493 100*requiredTemplateFraction))
494 raise lsst.pipe.base.NoWorkFound(message)
495
496
497def _subtractImages(science, template, backgroundModel=None):
498 """Subtract template from science, propagating relevant metadata.
499
500 Parameters
501 ----------
502 science : `lsst.afw.Exposure`
503 The input science image.
504 template : `lsst.afw.Exposure`
505 The template to subtract from the science image.
506 backgroundModel : `lsst.afw.MaskedImage`, optional
507 Differential background model
508
509 Returns
510 -------
511 difference : `lsst.afw.Exposure`
512 The subtracted image.
513 """
514 difference = science.clone()
515 if backgroundModel is not None:
516 difference.maskedImage -= backgroundModel
517 difference.maskedImage -= template.maskedImage
518 return difference
def finalize(self, template, science, difference, kernel, templateMatched=True, preConvMode=False, preConvKernel=None, spatiallyVarying=False)
def runConvolveScience(self, template, science, sources)
def runConvolveTemplate(self, template, science, sources)
def _convolveExposure(exposure, kernel, convolutionControl, bbox=None, psf=None)
void convolve(OutImageT &convolvedImage, InImageT const &inImage, KernelT const &kernel, ConvolutionControl const &convolutionControl=ConvolutionControl())
def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.)
def getPsfFwhm(psf)
Definition: utils.py:1083