Coverage for python/lsst/ip/diffim/subtractImages.py: 25%
252 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 03:35 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-06 03:35 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
24import lsst.afw.image
25import lsst.afw.math
26import lsst.geom
27from lsst.ip.diffim.utils import evaluateMeanPsfFwhm, getPsfFwhm
28from lsst.meas.algorithms import ScaleVarianceTask
29import lsst.pex.config
30import lsst.pipe.base
31from lsst.pex.exceptions import InvalidParameterError
32from lsst.pipe.base import connectionTypes
33from . import MakeKernelTask, DecorrelateALKernelTask
34from lsst.utils.timer import timeMethod
36__all__ = ["AlardLuptonSubtractConfig", "AlardLuptonSubtractTask",
37 "AlardLuptonPreconvolveSubtractConfig", "AlardLuptonPreconvolveSubtractTask"]
39_dimensions = ("instrument", "visit", "detector")
40_defaultTemplates = {"coaddName": "deep", "fakesType": ""}
43class SubtractInputConnections(lsst.pipe.base.PipelineTaskConnections,
44 dimensions=_dimensions,
45 defaultTemplates=_defaultTemplates):
46 template = connectionTypes.Input(
47 doc="Input warped template to subtract.",
48 dimensions=("instrument", "visit", "detector"),
49 storageClass="ExposureF",
50 name="{fakesType}{coaddName}Diff_templateExp"
51 )
52 science = connectionTypes.Input(
53 doc="Input science exposure to subtract from.",
54 dimensions=("instrument", "visit", "detector"),
55 storageClass="ExposureF",
56 name="{fakesType}calexp"
57 )
58 sources = connectionTypes.Input(
59 doc="Sources measured on the science exposure; "
60 "used to select sources for making the matching kernel.",
61 dimensions=("instrument", "visit", "detector"),
62 storageClass="SourceCatalog",
63 name="{fakesType}src"
64 )
65 finalizedPsfApCorrCatalog = connectionTypes.Input(
66 doc=("Per-visit finalized psf models and aperture correction maps. "
67 "These catalogs use the detector id for the catalog id, "
68 "sorted on id for fast lookup."),
69 dimensions=("instrument", "visit"),
70 storageClass="ExposureCatalog",
71 name="finalVisitSummary",
72 )
74 def __init__(self, *, config=None):
75 super().__init__(config=config)
76 if not config.doApplyFinalizedPsf:
77 self.inputs.remove("finalizedPsfApCorrCatalog")
80class SubtractImageOutputConnections(lsst.pipe.base.PipelineTaskConnections,
81 dimensions=_dimensions,
82 defaultTemplates=_defaultTemplates):
83 difference = connectionTypes.Output(
84 doc="Result of subtracting convolved template from science image.",
85 dimensions=("instrument", "visit", "detector"),
86 storageClass="ExposureF",
87 name="{fakesType}{coaddName}Diff_differenceTempExp",
88 )
89 matchedTemplate = connectionTypes.Output(
90 doc="Warped and PSF-matched template used to create `subtractedExposure`.",
91 dimensions=("instrument", "visit", "detector"),
92 storageClass="ExposureF",
93 name="{fakesType}{coaddName}Diff_matchedExp",
94 )
97class SubtractScoreOutputConnections(lsst.pipe.base.PipelineTaskConnections,
98 dimensions=_dimensions,
99 defaultTemplates=_defaultTemplates):
100 scoreExposure = connectionTypes.Output(
101 doc="The maximum likelihood image, used for the detection of diaSources.",
102 dimensions=("instrument", "visit", "detector"),
103 storageClass="ExposureF",
104 name="{fakesType}{coaddName}Diff_scoreExp",
105 )
108class AlardLuptonSubtractConnections(SubtractInputConnections, SubtractImageOutputConnections):
109 pass
112class AlardLuptonSubtractBaseConfig(lsst.pex.config.Config):
113 makeKernel = lsst.pex.config.ConfigurableField(
114 target=MakeKernelTask,
115 doc="Task to construct a matching kernel for convolution.",
116 )
117 doDecorrelation = lsst.pex.config.Field(
118 dtype=bool,
119 default=True,
120 doc="Perform diffim decorrelation to undo pixel correlation due to A&L "
121 "kernel convolution? If True, also update the diffim PSF."
122 )
123 decorrelate = lsst.pex.config.ConfigurableField(
124 target=DecorrelateALKernelTask,
125 doc="Task to decorrelate the image difference.",
126 )
127 requiredTemplateFraction = lsst.pex.config.Field(
128 dtype=float,
129 default=0.1,
130 doc="Abort task if template covers less than this fraction of pixels."
131 " Setting to 0 will always attempt image subtraction."
132 )
133 doScaleVariance = lsst.pex.config.Field(
134 dtype=bool,
135 default=True,
136 doc="Scale variance of the image difference?"
137 )
138 scaleVariance = lsst.pex.config.ConfigurableField(
139 target=ScaleVarianceTask,
140 doc="Subtask to rescale the variance of the template to the statistically expected level."
141 )
142 doSubtractBackground = lsst.pex.config.Field(
143 doc="Subtract the background fit when solving the kernel?",
144 dtype=bool,
145 default=True,
146 )
147 doApplyFinalizedPsf = lsst.pex.config.Field(
148 doc="Replace science Exposure's psf and aperture correction map"
149 " with those in finalizedPsfApCorrCatalog.",
150 dtype=bool,
151 default=False,
152 )
153 detectionThreshold = lsst.pex.config.Field(
154 dtype=float,
155 default=10,
156 doc="Minimum signal to noise ratio of detected sources "
157 "to use for calculating the PSF matching kernel."
158 )
159 badSourceFlags = lsst.pex.config.ListField(
160 dtype=str,
161 doc="Flags that, if set, the associated source should not "
162 "be used to determine the PSF matching kernel.",
163 default=("sky_source", "slot_Centroid_flag",
164 "slot_ApFlux_flag", "slot_PsfFlux_flag", ),
165 )
166 badMaskPlanes = lsst.pex.config.ListField(
167 dtype=str,
168 default=("NO_DATA", "BAD", "SAT", "EDGE"),
169 doc="Mask planes to exclude when selecting sources for PSF matching."
170 )
171 preserveTemplateMask = lsst.pex.config.ListField(
172 dtype=str,
173 default=("NO_DATA", "BAD", "SAT"),
174 doc="Mask planes from the template to propagate to the image difference."
175 )
177 def setDefaults(self):
178 self.makeKernel.kernel.name = "AL"
179 self.makeKernel.kernel.active.fitForBackground = self.doSubtractBackground
180 self.makeKernel.kernel.active.spatialKernelOrder = 1
181 self.makeKernel.kernel.active.spatialBgOrder = 2
184class AlardLuptonSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
185 pipelineConnections=AlardLuptonSubtractConnections):
186 mode = lsst.pex.config.ChoiceField(
187 dtype=str,
188 default="convolveTemplate",
189 allowed={"auto": "Choose which image to convolve at runtime.",
190 "convolveScience": "Only convolve the science image.",
191 "convolveTemplate": "Only convolve the template image."},
192 doc="Choose which image to convolve at runtime, or require that a specific image is convolved."
193 )
196class AlardLuptonSubtractTask(lsst.pipe.base.PipelineTask):
197 """Compute the image difference of a science and template image using
198 the Alard & Lupton (1998) algorithm.
199 """
200 ConfigClass = AlardLuptonSubtractConfig
201 _DefaultName = "alardLuptonSubtract"
203 def __init__(self, **kwargs):
204 super().__init__(**kwargs)
205 self.makeSubtask("decorrelate")
206 self.makeSubtask("makeKernel")
207 if self.config.doScaleVariance:
208 self.makeSubtask("scaleVariance")
210 self.convolutionControl = lsst.afw.math.ConvolutionControl()
211 # Normalization is an extra, unnecessary, calculation and will result
212 # in mis-subtraction of the images if there are calibration errors.
213 self.convolutionControl.setDoNormalize(False)
214 self.convolutionControl.setDoCopyEdge(True)
216 def _applyExternalCalibrations(self, exposure, finalizedPsfApCorrCatalog):
217 """Replace calibrations (psf, and ApCorrMap) on this exposure with external ones.".
219 Parameters
220 ----------
221 exposure : `lsst.afw.image.exposure.Exposure`
222 Input exposure to adjust calibrations.
223 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`
224 Exposure catalog with finalized psf models and aperture correction
225 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
226 the detector id for the catalog id, sorted on id for fast lookup.
228 Returns
229 -------
230 exposure : `lsst.afw.image.exposure.Exposure`
231 Exposure with adjusted calibrations.
232 """
233 detectorId = exposure.info.getDetector().getId()
235 row = finalizedPsfApCorrCatalog.find(detectorId)
236 if row is None:
237 self.log.warning("Detector id %s not found in finalizedPsfApCorrCatalog; "
238 "Using original psf.", detectorId)
239 else:
240 psf = row.getPsf()
241 apCorrMap = row.getApCorrMap()
242 if psf is None:
243 self.log.warning("Detector id %s has None for psf in "
244 "finalizedPsfApCorrCatalog; Using original psf and aperture correction.",
245 detectorId)
246 elif apCorrMap is None:
247 self.log.warning("Detector id %s has None for apCorrMap in "
248 "finalizedPsfApCorrCatalog; Using original psf and aperture correction.",
249 detectorId)
250 else:
251 exposure.setPsf(psf)
252 exposure.info.setApCorrMap(apCorrMap)
254 return exposure
256 @timeMethod
257 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None):
258 """PSF match, subtract, and decorrelate two images.
260 Parameters
261 ----------
262 template : `lsst.afw.image.ExposureF`
263 Template exposure, warped to match the science exposure.
264 science : `lsst.afw.image.ExposureF`
265 Science exposure to subtract from the template.
266 sources : `lsst.afw.table.SourceCatalog`
267 Identified sources on the science exposure. This catalog is used to
268 select sources in order to perform the AL PSF matching on stamp
269 images around them.
270 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
271 Exposure catalog with finalized psf models and aperture correction
272 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
273 the detector id for the catalog id, sorted on id for fast lookup.
275 Returns
276 -------
277 results : `lsst.pipe.base.Struct`
278 ``difference`` : `lsst.afw.image.ExposureF`
279 Result of subtracting template and science.
280 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
281 Warped and PSF-matched template exposure.
282 ``backgroundModel`` : `lsst.afw.math.Function2D`
283 Background model that was fit while solving for the PSF-matching kernel
284 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
285 Kernel used to PSF-match the convolved image.
287 Raises
288 ------
289 RuntimeError
290 If an unsupported convolution mode is supplied.
291 RuntimeError
292 If there are too few sources to calculate the PSF matching kernel.
293 lsst.pipe.base.NoWorkFound
294 Raised if fraction of good pixels, defined as not having NO_DATA
295 set, is less then the configured requiredTemplateFraction
296 """
297 self._prepareInputs(template, science,
298 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
300 # In the event that getPsfFwhm fails, evaluate the PSF on a grid.
301 fwhmExposureBuffer = self.config.makeKernel.fwhmExposureBuffer
302 fwhmExposureGrid = self.config.makeKernel.fwhmExposureGrid
304 # Calling getPsfFwhm on template.psf fails on some rare occasions when
305 # the template has no input exposures at the average position of the
306 # stars. So we try getPsfFwhm first on template, and if that fails we
307 # evaluate the PSF on a grid specified by fwhmExposure* fields.
308 # To keep consistent definitions for PSF size on the template and
309 # science images, we use the same method for both.
310 try:
311 templatePsfSize = getPsfFwhm(template.psf)
312 sciencePsfSize = getPsfFwhm(science.psf)
313 except InvalidParameterError:
314 self.log.info("Unable to evaluate PSF at the average position. "
315 "Evaluting PSF on a grid of points."
316 )
317 templatePsfSize = evaluateMeanPsfFwhm(template,
318 fwhmExposureBuffer=fwhmExposureBuffer,
319 fwhmExposureGrid=fwhmExposureGrid
320 )
321 sciencePsfSize = evaluateMeanPsfFwhm(science,
322 fwhmExposureBuffer=fwhmExposureBuffer,
323 fwhmExposureGrid=fwhmExposureGrid
324 )
325 self.log.info("Science PSF FWHM: %f pixels", sciencePsfSize)
326 self.log.info("Template PSF FWHM: %f pixels", templatePsfSize)
327 selectSources = self._sourceSelector(sources, science.mask)
329 if self.config.mode == "auto":
330 convolveTemplate = _shapeTest(template,
331 science,
332 fwhmExposureBuffer=fwhmExposureBuffer,
333 fwhmExposureGrid=fwhmExposureGrid)
334 if convolveTemplate:
335 if sciencePsfSize < templatePsfSize:
336 self.log.info("Average template PSF size is greater, "
337 "but science PSF greater in one dimension: convolving template image.")
338 else:
339 self.log.info("Science PSF size is greater: convolving template image.")
340 else:
341 self.log.info("Template PSF size is greater: convolving science image.")
342 elif self.config.mode == "convolveTemplate":
343 self.log.info("`convolveTemplate` is set: convolving template image.")
344 convolveTemplate = True
345 elif self.config.mode == "convolveScience":
346 self.log.info("`convolveScience` is set: convolving science image.")
347 convolveTemplate = False
348 else:
349 raise RuntimeError("Cannot handle AlardLuptonSubtract mode: %s", self.config.mode)
351 if convolveTemplate:
352 subtractResults = self.runConvolveTemplate(template, science, selectSources)
353 else:
354 subtractResults = self.runConvolveScience(template, science, selectSources)
356 return subtractResults
358 def runConvolveTemplate(self, template, science, selectSources):
359 """Convolve the template image with a PSF-matching kernel and subtract
360 from the science image.
362 Parameters
363 ----------
364 template : `lsst.afw.image.ExposureF`
365 Template exposure, warped to match the science exposure.
366 science : `lsst.afw.image.ExposureF`
367 Science exposure to subtract from the template.
368 selectSources : `lsst.afw.table.SourceCatalog`
369 Identified sources on the science exposure. This catalog is used to
370 select sources in order to perform the AL PSF matching on stamp
371 images around them.
373 Returns
374 -------
375 results : `lsst.pipe.base.Struct`
377 ``difference`` : `lsst.afw.image.ExposureF`
378 Result of subtracting template and science.
379 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
380 Warped and PSF-matched template exposure.
381 ``backgroundModel`` : `lsst.afw.math.Function2D`
382 Background model that was fit while solving for the PSF-matching kernel
383 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
384 Kernel used to PSF-match the template to the science image.
385 """
386 kernelSources = self.makeKernel.selectKernelSources(template, science,
387 candidateList=selectSources,
388 preconvolved=False)
389 kernelResult = self.makeKernel.run(template, science, kernelSources,
390 preconvolved=False)
392 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
393 self.convolutionControl,
394 bbox=science.getBBox(),
395 psf=science.psf,
396 photoCalib=science.photoCalib)
398 difference = _subtractImages(science, matchedTemplate,
399 backgroundModel=(kernelResult.backgroundModel
400 if self.config.doSubtractBackground else None))
401 correctedExposure = self.finalize(template, science, difference,
402 kernelResult.psfMatchingKernel,
403 templateMatched=True)
405 return lsst.pipe.base.Struct(difference=correctedExposure,
406 matchedTemplate=matchedTemplate,
407 matchedScience=science,
408 backgroundModel=kernelResult.backgroundModel,
409 psfMatchingKernel=kernelResult.psfMatchingKernel)
411 def runConvolveScience(self, template, science, selectSources):
412 """Convolve the science image with a PSF-matching kernel and subtract the template image.
414 Parameters
415 ----------
416 template : `lsst.afw.image.ExposureF`
417 Template exposure, warped to match the science exposure.
418 science : `lsst.afw.image.ExposureF`
419 Science exposure to subtract from the template.
420 selectSources : `lsst.afw.table.SourceCatalog`
421 Identified sources on the science exposure. This catalog is used to
422 select sources in order to perform the AL PSF matching on stamp
423 images around them.
425 Returns
426 -------
427 results : `lsst.pipe.base.Struct`
429 ``difference`` : `lsst.afw.image.ExposureF`
430 Result of subtracting template and science.
431 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
432 Warped template exposure. Note that in this case, the template
433 is not PSF-matched to the science image.
434 ``backgroundModel`` : `lsst.afw.math.Function2D`
435 Background model that was fit while solving for the PSF-matching kernel
436 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
437 Kernel used to PSF-match the science image to the template.
438 """
439 bbox = science.getBBox()
440 kernelSources = self.makeKernel.selectKernelSources(science, template,
441 candidateList=selectSources,
442 preconvolved=False)
443 kernelResult = self.makeKernel.run(science, template, kernelSources,
444 preconvolved=False)
445 modelParams = kernelResult.backgroundModel.getParameters()
446 # We must invert the background model if the matching kernel is solved for the science image.
447 kernelResult.backgroundModel.setParameters([-p for p in modelParams])
449 kernelImage = lsst.afw.image.ImageD(kernelResult.psfMatchingKernel.getDimensions())
450 norm = kernelResult.psfMatchingKernel.computeImage(kernelImage, doNormalize=False)
452 matchedScience = self._convolveExposure(science, kernelResult.psfMatchingKernel,
453 self.convolutionControl,
454 psf=template.psf)
456 # Place back on native photometric scale
457 matchedScience.maskedImage /= norm
458 matchedTemplate = template.clone()[bbox]
459 matchedTemplate.maskedImage /= norm
460 matchedTemplate.setPhotoCalib(science.photoCalib)
462 difference = _subtractImages(matchedScience, matchedTemplate,
463 backgroundModel=(kernelResult.backgroundModel
464 if self.config.doSubtractBackground else None))
466 correctedExposure = self.finalize(template, science, difference,
467 kernelResult.psfMatchingKernel,
468 templateMatched=False)
470 return lsst.pipe.base.Struct(difference=correctedExposure,
471 matchedTemplate=matchedTemplate,
472 matchedScience=matchedScience,
473 backgroundModel=kernelResult.backgroundModel,
474 psfMatchingKernel=kernelResult.psfMatchingKernel,)
476 def finalize(self, template, science, difference, kernel,
477 templateMatched=True,
478 preConvMode=False,
479 preConvKernel=None,
480 spatiallyVarying=False):
481 """Decorrelate the difference image to undo the noise correlations
482 caused by convolution.
484 Parameters
485 ----------
486 template : `lsst.afw.image.ExposureF`
487 Template exposure, warped to match the science exposure.
488 science : `lsst.afw.image.ExposureF`
489 Science exposure to subtract from the template.
490 difference : `lsst.afw.image.ExposureF`
491 Result of subtracting template and science.
492 kernel : `lsst.afw.math.Kernel`
493 An (optionally spatially-varying) PSF matching kernel
494 templateMatched : `bool`, optional
495 Was the template PSF-matched to the science image?
496 preConvMode : `bool`, optional
497 Was the science image preconvolved with its own PSF
498 before PSF matching the template?
499 preConvKernel : `lsst.afw.detection.Psf`, optional
500 If not `None`, then the science image was pre-convolved with
501 (the reflection of) this kernel. Must be normalized to sum to 1.
502 spatiallyVarying : `bool`, optional
503 Compute the decorrelation kernel spatially varying across the image?
505 Returns
506 -------
507 correctedExposure : `lsst.afw.image.ExposureF`
508 The decorrelated image difference.
509 """
510 # Erase existing detection mask planes.
511 # We don't want the detection mask from the science image
512 mask = difference.mask
513 mask &= ~(mask.getPlaneBitMask("DETECTED") | mask.getPlaneBitMask("DETECTED_NEGATIVE"))
515 # We have cleared the template mask plane, so copy the mask plane of
516 # the image difference so that we can calculate correct statistics
517 # during decorrelation. Do this regardless of whether decorrelation is
518 # used for consistency.
519 template[science.getBBox()].mask.array[...] = difference.mask.array[...]
520 if self.config.doDecorrelation:
521 self.log.info("Decorrelating image difference.")
522 # We have cleared the template mask plane, so copy the mask plane of
523 # the image difference so that we can calculate correct statistics
524 # during decorrelation
525 template[science.getBBox()].mask.array[...] = difference.mask.array[...]
526 correctedExposure = self.decorrelate.run(science, template[science.getBBox()], difference, kernel,
527 templateMatched=templateMatched,
528 preConvMode=preConvMode,
529 preConvKernel=preConvKernel,
530 spatiallyVarying=spatiallyVarying).correctedExposure
531 else:
532 self.log.info("NOT decorrelating image difference.")
533 correctedExposure = difference
534 return correctedExposure
536 @staticmethod
537 def _validateExposures(template, science):
538 """Check that the WCS of the two Exposures match, and the template bbox
539 contains the science bbox.
541 Parameters
542 ----------
543 template : `lsst.afw.image.ExposureF`
544 Template exposure, warped to match the science exposure.
545 science : `lsst.afw.image.ExposureF`
546 Science exposure to subtract from the template.
548 Raises
549 ------
550 AssertionError
551 Raised if the WCS of the template is not equal to the science WCS,
552 or if the science image is not fully contained in the template
553 bounding box.
554 """
555 assert template.wcs == science.wcs,\
556 "Template and science exposure WCS are not identical."
557 templateBBox = template.getBBox()
558 scienceBBox = science.getBBox()
560 assert templateBBox.contains(scienceBBox),\
561 "Template bbox does not contain all of the science image."
563 @staticmethod
564 def _convolveExposure(exposure, kernel, convolutionControl,
565 bbox=None,
566 psf=None,
567 photoCalib=None):
568 """Convolve an exposure with the given kernel.
570 Parameters
571 ----------
572 exposure : `lsst.afw.Exposure`
573 exposure to convolve.
574 kernel : `lsst.afw.math.LinearCombinationKernel`
575 PSF matching kernel computed in the ``makeKernel`` subtask.
576 convolutionControl : `lsst.afw.math.ConvolutionControl`
577 Configuration for convolve algorithm.
578 bbox : `lsst.geom.Box2I`, optional
579 Bounding box to trim the convolved exposure to.
580 psf : `lsst.afw.detection.Psf`, optional
581 Point spread function (PSF) to set for the convolved exposure.
582 photoCalib : `lsst.afw.image.PhotoCalib`, optional
583 Photometric calibration of the convolved exposure.
585 Returns
586 -------
587 convolvedExp : `lsst.afw.Exposure`
588 The convolved image.
589 """
590 convolvedExposure = exposure.clone()
591 if psf is not None:
592 convolvedExposure.setPsf(psf)
593 if photoCalib is not None:
594 convolvedExposure.setPhotoCalib(photoCalib)
595 convolvedImage = lsst.afw.image.MaskedImageF(exposure.getBBox())
596 lsst.afw.math.convolve(convolvedImage, exposure.maskedImage, kernel, convolutionControl)
597 convolvedExposure.setMaskedImage(convolvedImage)
598 if bbox is None:
599 return convolvedExposure
600 else:
601 return convolvedExposure[bbox]
603 def _sourceSelector(self, sources, mask):
604 """Select sources from a catalog that meet the selection criteria.
606 Parameters
607 ----------
608 sources : `lsst.afw.table.SourceCatalog`
609 Input source catalog to select sources from.
610 mask : `lsst.afw.image.Mask`
611 The image mask plane to use to reject sources
612 based on their location on the ccd.
614 Returns
615 -------
616 selectSources : `lsst.afw.table.SourceCatalog`
617 The input source catalog, with flagged and low signal-to-noise
618 sources removed.
620 Raises
621 ------
622 RuntimeError
623 If there are too few sources to compute the PSF matching kernel
624 remaining after source selection.
625 """
626 flags = np.ones(len(sources), dtype=bool)
627 for flag in self.config.badSourceFlags:
628 try:
629 flags *= ~sources[flag]
630 except Exception as e:
631 self.log.warning("Could not apply source flag: %s", e)
632 sToNFlag = (sources.getPsfInstFlux()/sources.getPsfInstFluxErr()) > self.config.detectionThreshold
633 flags *= sToNFlag
634 flags *= self._checkMask(mask, sources, self.config.badMaskPlanes)
635 selectSources = sources[flags]
636 self.log.info("%i/%i=%.1f%% of sources selected for PSF matching from the input catalog",
637 len(selectSources), len(sources), 100*len(selectSources)/len(sources))
638 if len(selectSources) < self.config.makeKernel.nStarPerCell:
639 self.log.error("Too few sources to calculate the PSF matching kernel: "
640 "%i selected but %i needed for the calculation.",
641 len(selectSources), self.config.makeKernel.nStarPerCell)
642 raise RuntimeError("Cannot compute PSF matching kernel: too few sources selected.")
644 return selectSources.copy(deep=True)
646 @staticmethod
647 def _checkMask(mask, sources, badMaskPlanes):
648 """Exclude sources that are located on masked pixels.
650 Parameters
651 ----------
652 mask : `lsst.afw.image.Mask`
653 The image mask plane to use to reject sources
654 based on the location of their centroid on the ccd.
655 sources : `lsst.afw.table.SourceCatalog`
656 The source catalog to evaluate.
657 badMaskPlanes : `list` of `str`
658 List of the names of the mask planes to exclude.
660 Returns
661 -------
662 flags : `numpy.ndarray` of `bool`
663 Array indicating whether each source in the catalog should be
664 kept (True) or rejected (False) based on the value of the
665 mask plane at its location.
666 """
667 badPixelMask = lsst.afw.image.Mask.getPlaneBitMask(badMaskPlanes)
668 xv = np.rint(sources.getX() - mask.getX0())
669 yv = np.rint(sources.getY() - mask.getY0())
671 mv = mask.array[yv.astype(int), xv.astype(int)]
672 flags = np.bitwise_and(mv, badPixelMask) == 0
673 return flags
675 def _prepareInputs(self, template, science,
676 finalizedPsfApCorrCatalog=None):
677 """Perform preparatory calculations common to all Alard&Lupton Tasks.
679 Parameters
680 ----------
681 template : `lsst.afw.image.ExposureF`
682 Template exposure, warped to match the science exposure.
683 The variance plane of the template image is modified in place.
684 science : `lsst.afw.image.ExposureF`
685 Science exposure to subtract from the template.
686 The variance plane of the science image is modified in place.
687 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
688 Exposure catalog with finalized psf models and aperture correction
689 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
690 the detector id for the catalog id, sorted on id for fast lookup.
691 """
692 self._validateExposures(template, science)
693 if self.config.doApplyFinalizedPsf:
694 self._applyExternalCalibrations(science,
695 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
696 checkTemplateIsSufficient(template, self.log,
697 requiredTemplateFraction=self.config.requiredTemplateFraction)
699 if self.config.doScaleVariance:
700 # Scale the variance of the template and science images before
701 # convolution, subtraction, or decorrelation so that they have the
702 # correct ratio.
703 templateVarFactor = self.scaleVariance.run(template.maskedImage)
704 sciVarFactor = self.scaleVariance.run(science.maskedImage)
705 self.log.info("Template variance scaling factor: %.2f", templateVarFactor)
706 self.metadata.add("scaleTemplateVarianceFactor", templateVarFactor)
707 self.log.info("Science variance scaling factor: %.2f", sciVarFactor)
708 self.metadata.add("scaleScienceVarianceFactor", sciVarFactor)
709 self._clearMask(template)
711 def _clearMask(self, template):
712 """Clear the mask plane of the template.
714 Parameters
715 ----------
716 template : `lsst.afw.image.ExposureF`
717 Template exposure, warped to match the science exposure.
718 The mask plane will be modified in place.
719 """
720 mask = template.mask
721 clearMaskPlanes = [maskplane for maskplane in mask.getMaskPlaneDict().keys()
722 if maskplane not in self.config.preserveTemplateMask]
724 bitMaskToClear = mask.getPlaneBitMask(clearMaskPlanes)
725 mask &= ~bitMaskToClear
728class AlardLuptonPreconvolveSubtractConnections(SubtractInputConnections,
729 SubtractScoreOutputConnections):
730 pass
733class AlardLuptonPreconvolveSubtractConfig(AlardLuptonSubtractBaseConfig, lsst.pipe.base.PipelineTaskConfig,
734 pipelineConnections=AlardLuptonPreconvolveSubtractConnections):
735 pass
738class AlardLuptonPreconvolveSubtractTask(AlardLuptonSubtractTask):
739 """Subtract a template from a science image, convolving the science image
740 before computing the kernel, and also convolving the template before
741 subtraction.
742 """
743 ConfigClass = AlardLuptonPreconvolveSubtractConfig
744 _DefaultName = "alardLuptonPreconvolveSubtract"
746 def run(self, template, science, sources, finalizedPsfApCorrCatalog=None):
747 """Preconvolve the science image with its own PSF,
748 convolve the template image with a PSF-matching kernel and subtract
749 from the preconvolved science image.
751 Parameters
752 ----------
753 template : `lsst.afw.image.ExposureF`
754 The template image, which has previously been warped to
755 the science image. The template bbox will be padded by a few pixels
756 compared to the science bbox.
757 science : `lsst.afw.image.ExposureF`
758 The science exposure.
759 sources : `lsst.afw.table.SourceCatalog`
760 Identified sources on the science exposure. This catalog is used to
761 select sources in order to perform the AL PSF matching on stamp
762 images around them.
763 finalizedPsfApCorrCatalog : `lsst.afw.table.ExposureCatalog`, optional
764 Exposure catalog with finalized psf models and aperture correction
765 maps to be applied if config.doApplyFinalizedPsf=True. Catalog uses
766 the detector id for the catalog id, sorted on id for fast lookup.
768 Returns
769 -------
770 results : `lsst.pipe.base.Struct`
771 ``scoreExposure`` : `lsst.afw.image.ExposureF`
772 Result of subtracting the convolved template and science images.
773 Attached PSF is that of the original science image.
774 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
775 Warped and PSF-matched template exposure.
776 Attached PSF is that of the original science image.
777 ``matchedScience`` : `lsst.afw.image.ExposureF`
778 The science exposure after convolving with its own PSF.
779 Attached PSF is that of the original science image.
780 ``backgroundModel`` : `lsst.afw.math.Function2D`
781 Background model that was fit while solving for the PSF-matching kernel
782 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
783 Final kernel used to PSF-match the template to the science image.
784 """
785 self._prepareInputs(template, science,
786 finalizedPsfApCorrCatalog=finalizedPsfApCorrCatalog)
788 # TODO: DM-37212 we need to mirror the kernel in order to get correct cross correlation
789 scienceKernel = science.psf.getKernel()
790 matchedScience = self._convolveExposure(science, scienceKernel, self.convolutionControl)
791 selectSources = self._sourceSelector(sources, matchedScience.mask)
793 subtractResults = self.runPreconvolve(template, science, matchedScience, selectSources, scienceKernel)
795 return subtractResults
797 def runPreconvolve(self, template, science, matchedScience, selectSources, preConvKernel):
798 """Convolve the science image with its own PSF, then convolve the
799 template with a matching kernel and subtract to form the Score exposure.
801 Parameters
802 ----------
803 template : `lsst.afw.image.ExposureF`
804 Template exposure, warped to match the science exposure.
805 science : `lsst.afw.image.ExposureF`
806 Science exposure to subtract from the template.
807 matchedScience : `lsst.afw.image.ExposureF`
808 The science exposure, convolved with the reflection of its own PSF.
809 selectSources : `lsst.afw.table.SourceCatalog`
810 Identified sources on the science exposure. This catalog is used to
811 select sources in order to perform the AL PSF matching on stamp
812 images around them.
813 preConvKernel : `lsst.afw.math.Kernel`
814 The reflection of the kernel that was used to preconvolve
815 the `science` exposure.
816 Must be normalized to sum to 1.
818 Returns
819 -------
820 results : `lsst.pipe.base.Struct`
822 ``scoreExposure`` : `lsst.afw.image.ExposureF`
823 Result of subtracting the convolved template and science images.
824 Attached PSF is that of the original science image.
825 ``matchedTemplate`` : `lsst.afw.image.ExposureF`
826 Warped and PSF-matched template exposure.
827 Attached PSF is that of the original science image.
828 ``matchedScience`` : `lsst.afw.image.ExposureF`
829 The science exposure after convolving with its own PSF.
830 Attached PSF is that of the original science image.
831 ``backgroundModel`` : `lsst.afw.math.Function2D`
832 Background model that was fit while solving for the PSF-matching kernel
833 ``psfMatchingKernel`` : `lsst.afw.math.Kernel`
834 Final kernel used to PSF-match the template to the science image.
835 """
836 bbox = science.getBBox()
837 innerBBox = preConvKernel.shrinkBBox(bbox)
839 kernelSources = self.makeKernel.selectKernelSources(template[innerBBox], matchedScience[innerBBox],
840 candidateList=selectSources,
841 preconvolved=True)
842 kernelResult = self.makeKernel.run(template[innerBBox], matchedScience[innerBBox], kernelSources,
843 preconvolved=True)
845 matchedTemplate = self._convolveExposure(template, kernelResult.psfMatchingKernel,
846 self.convolutionControl,
847 bbox=bbox,
848 psf=science.psf,
849 photoCalib=science.photoCalib)
850 score = _subtractImages(matchedScience, matchedTemplate,
851 backgroundModel=(kernelResult.backgroundModel
852 if self.config.doSubtractBackground else None))
853 correctedScore = self.finalize(template[bbox], science, score,
854 kernelResult.psfMatchingKernel,
855 templateMatched=True, preConvMode=True,
856 preConvKernel=preConvKernel)
858 return lsst.pipe.base.Struct(scoreExposure=correctedScore,
859 matchedTemplate=matchedTemplate,
860 matchedScience=matchedScience,
861 backgroundModel=kernelResult.backgroundModel,
862 psfMatchingKernel=kernelResult.psfMatchingKernel)
865def checkTemplateIsSufficient(templateExposure, logger, requiredTemplateFraction=0.):
866 """Raise NoWorkFound if template coverage < requiredTemplateFraction
868 Parameters
869 ----------
870 templateExposure : `lsst.afw.image.ExposureF`
871 The template exposure to check
872 logger : `lsst.log.Log`
873 Logger for printing output.
874 requiredTemplateFraction : `float`, optional
875 Fraction of pixels of the science image required to have coverage
876 in the template.
878 Raises
879 ------
880 lsst.pipe.base.NoWorkFound
881 Raised if fraction of good pixels, defined as not having NO_DATA
882 set, is less then the configured requiredTemplateFraction
883 """
884 # Count the number of pixels with the NO_DATA mask bit set
885 # counting NaN pixels is insufficient because pixels without data are often intepolated over)
886 pixNoData = np.count_nonzero(templateExposure.mask.array
887 & templateExposure.mask.getPlaneBitMask('NO_DATA'))
888 pixGood = templateExposure.getBBox().getArea() - pixNoData
889 logger.info("template has %d good pixels (%.1f%%)", pixGood,
890 100*pixGood/templateExposure.getBBox().getArea())
892 if pixGood/templateExposure.getBBox().getArea() < requiredTemplateFraction:
893 message = ("Insufficient Template Coverage. (%.1f%% < %.1f%%) Not attempting subtraction. "
894 "To force subtraction, set config requiredTemplateFraction=0." % (
895 100*pixGood/templateExposure.getBBox().getArea(),
896 100*requiredTemplateFraction))
897 raise lsst.pipe.base.NoWorkFound(message)
900def _subtractImages(science, template, backgroundModel=None):
901 """Subtract template from science, propagating relevant metadata.
903 Parameters
904 ----------
905 science : `lsst.afw.Exposure`
906 The input science image.
907 template : `lsst.afw.Exposure`
908 The template to subtract from the science image.
909 backgroundModel : `lsst.afw.MaskedImage`, optional
910 Differential background model
912 Returns
913 -------
914 difference : `lsst.afw.Exposure`
915 The subtracted image.
916 """
917 difference = science.clone()
918 if backgroundModel is not None:
919 difference.maskedImage -= backgroundModel
920 difference.maskedImage -= template.maskedImage
921 return difference
924def _shapeTest(exp1, exp2, fwhmExposureBuffer, fwhmExposureGrid):
925 """Determine that the PSF of ``exp1`` is not wider than that of ``exp2``.
927 Parameters
928 ----------
929 exp1 : `~lsst.afw.image.Exposure`
930 Exposure with the reference point spread function (PSF) to evaluate.
931 exp2 : `~lsst.afw.image.Exposure`
932 Exposure with a candidate point spread function (PSF) to evaluate.
933 fwhmExposureBuffer : `float`
934 Fractional buffer margin to be left out of all sides of the image
935 during the construction of the grid to compute mean PSF FWHM in an
936 exposure, if the PSF is not available at its average position.
937 fwhmExposureGrid : `int`
938 Grid size to compute the mean FWHM in an exposure, if the PSF is not
939 available at its average position.
940 Returns
941 -------
942 result : `bool`
943 True if ``exp1`` has a PSF that is not wider than that of ``exp2`` in
944 either dimension.
945 """
946 try:
947 shape1 = getPsfFwhm(exp1.psf, average=False)
948 shape2 = getPsfFwhm(exp2.psf, average=False)
949 except InvalidParameterError:
950 shape1 = evaluateMeanPsfFwhm(exp1,
951 fwhmExposureBuffer=fwhmExposureBuffer,
952 fwhmExposureGrid=fwhmExposureGrid
953 )
954 shape2 = evaluateMeanPsfFwhm(exp2,
955 fwhmExposureBuffer=fwhmExposureBuffer,
956 fwhmExposureGrid=fwhmExposureGrid
957 )
958 return shape1 <= shape2
960 # Results from getPsfFwhm is a tuple of two values, one for each dimension.
961 xTest = shape1[0] <= shape2[0]
962 yTest = shape1[1] <= shape2[1]
963 return xTest | yTest