Coverage for tests/test_subtractTask.py: 6%
747 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-08 03:38 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-08 03:38 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import unittest
24import lsst.afw.math as afwMath
25import lsst.afw.table as afwTable
26import lsst.geom
27import lsst.meas.algorithms as measAlg
28from lsst.ip.diffim import subtractImages
29from lsst.pex.config import FieldValidationError
30from lsst.pipe.base import NoWorkFound
31import lsst.utils.tests
32import numpy as np
33from lsst.ip.diffim.utils import (computeRobustStatistics, computePSFNoiseEquivalentArea,
34 evaluateMeanPsfFwhm, getPsfFwhm, makeStats, makeTestImage)
35from lsst.pex.exceptions import InvalidParameterError
38class CustomCoaddPsf(measAlg.CoaddPsf):
39 """A custom CoaddPSF that overrides the getAveragePosition method.
40 """
41 def getAveragePosition(self):
42 return lsst.geom.Point2D(-10000, -10000)
45class AlardLuptonSubtractTest(lsst.utils.tests.TestCase):
47 def test_allowed_config_modes(self):
48 """Verify the allowable modes for convolution.
49 """
50 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
51 config.mode = 'auto'
52 config.mode = 'convolveScience'
53 config.mode = 'convolveTemplate'
55 with self.assertRaises(FieldValidationError):
56 config.mode = 'aotu'
58 def test_mismatched_template(self):
59 """Test that an error is raised if the template
60 does not fully contain the science image.
61 """
62 xSize = 200
63 ySize = 200
64 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20)
65 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
66 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
67 task = subtractImages.AlardLuptonSubtractTask(config=config)
68 with self.assertRaises(AssertionError):
69 task.run(template, science, sources)
71 def test_incomplete_template_coverage(self):
72 noiseLevel = 1.
73 border = 20
74 xSize = 400
75 ySize = 400
76 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6, nSrc=50,
77 xSize=xSize, ySize=ySize)
78 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7, nSrc=50,
79 templateBorderSize=border, doApplyCalibration=True,
80 xSize=xSize, ySize=ySize)
82 science_height = science.getBBox().getDimensions().getY()
84 def _run_and_check_coverage(template_coverage,
85 requiredTemplateFraction=0.1,
86 minTemplateFractionForExpectedSuccess=0.2):
87 template_cut = template.clone()
88 template_height = int(science_height*template_coverage + border)
89 template_cut.image.array[:, template_height:] = 0
90 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA')
91 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
92 config.requiredTemplateFraction = requiredTemplateFraction
93 config.minTemplateFractionForExpectedSuccess = minTemplateFractionForExpectedSuccess
94 if template_coverage < requiredTemplateFraction:
95 doRaise = True
96 elif template_coverage < minTemplateFractionForExpectedSuccess:
97 doRaise = True
98 else:
99 doRaise = False
100 task = subtractImages.AlardLuptonSubtractTask(config=config)
101 if doRaise:
102 with self.assertRaises(NoWorkFound):
103 task.run(template_cut, science.clone(), sources.copy(deep=True))
104 else:
105 task.run(template_cut, science.clone(), sources.copy(deep=True))
106 _run_and_check_coverage(template_coverage=0.09)
107 _run_and_check_coverage(template_coverage=0.19)
108 _run_and_check_coverage(template_coverage=0.7)
110 def test_clear_template_mask(self):
111 noiseLevel = 1.
112 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6)
113 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
114 templateBorderSize=20, doApplyCalibration=True)
115 diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"]
116 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
117 config.doSubtractBackground = False
118 config.mode = "convolveTemplate"
119 # Ensure that each each mask plane is set for some pixels
120 mask = template.mask
121 x0 = 50
122 x1 = 75
123 y0 = 150
124 y1 = 175
125 scienceMaskCheck = {}
126 for maskPlane in mask.getMaskPlaneDict().keys():
127 scienceMaskCheck[maskPlane] = np.sum(science.mask.array & mask.getPlaneBitMask(maskPlane) > 0)
128 mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane)
129 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0))
131 task = subtractImages.AlardLuptonSubtractTask(config=config)
132 output = task.run(template, science, sources)
133 # Verify that the template mask has been modified in place
134 for maskPlane in mask.getMaskPlaneDict().keys():
135 if maskPlane in diffimEmptyMaskPlanes:
136 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0))
137 elif maskPlane in config.preserveTemplateMask:
138 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0))
139 else:
140 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0))
141 # Mask planes set in the science image should also be set in the difference
142 # Except the "DETECTED" planes should have been cleared
143 diffimMask = output.difference.mask
144 for maskPlane, scienceSum in scienceMaskCheck.items():
145 diffimSum = np.sum(diffimMask.array & mask.getPlaneBitMask(maskPlane) > 0)
146 if maskPlane in diffimEmptyMaskPlanes:
147 self.assertEqual(diffimSum, 0)
148 else:
149 self.assertTrue(diffimSum >= scienceSum)
151 def test_equal_images(self):
152 """Test that running with enough sources produces reasonable output,
153 with the same size psf in the template and science.
154 """
155 noiseLevel = 1.
156 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6)
157 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7,
158 templateBorderSize=20, doApplyCalibration=True)
159 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
160 config.doSubtractBackground = False
161 task = subtractImages.AlardLuptonSubtractTask(config=config)
162 output = task.run(template, science, sources)
163 # There shoud be no NaN values in the difference image
164 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
165 # Mean of difference image should be close to zero.
166 meanError = noiseLevel/np.sqrt(output.difference.image.array.size)
167 # Make sure to include pixels with the DETECTED mask bit set.
168 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA", "DETECTED", "DETECTED_NEGATIVE"))
169 differenceMean = computeRobustStatistics(output.difference.image, output.difference.mask, statsCtrl)
170 self.assertFloatsAlmostEqual(differenceMean, 0, atol=5*meanError)
171 # stddev of difference image should be close to expected value.
172 differenceStd = computeRobustStatistics(output.difference.image, output.difference.mask,
173 makeStats(), statistic=afwMath.STDEV)
174 self.assertFloatsAlmostEqual(differenceStd, np.sqrt(2)*noiseLevel, rtol=0.1)
176 def test_equal_images_missing_mask_planes(self):
177 """Test that running with enough sources produces reasonable output,
178 with the same size psf in the template and science and with missing
179 mask planes.
180 """
181 noiseLevel = 1.
182 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6, addMaskPlanes=[])
183 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7,
184 templateBorderSize=20, doApplyCalibration=True, addMaskPlanes=[])
185 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
186 config.doSubtractBackground = False
187 task = subtractImages.AlardLuptonSubtractTask(config=config)
188 output = task.run(template, science, sources)
189 # There shoud be no NaN values in the difference image
190 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
191 # Mean of difference image should be close to zero.
192 meanError = noiseLevel/np.sqrt(output.difference.image.array.size)
193 # Make sure to include pixels with the DETECTED mask bit set.
194 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA", "DETECTED", "DETECTED_NEGATIVE"))
195 differenceMean = computeRobustStatistics(output.difference.image, output.difference.mask, statsCtrl)
196 self.assertFloatsAlmostEqual(differenceMean, 0, atol=5*meanError)
197 # stddev of difference image should be close to expected value.
198 differenceStd = computeRobustStatistics(output.difference.image, output.difference.mask,
199 makeStats(), statistic=afwMath.STDEV)
200 self.assertFloatsAlmostEqual(differenceStd, np.sqrt(2)*noiseLevel, rtol=0.1)
202 def test_psf_size(self):
203 """Test that the image subtract task runs without failing, if
204 fwhmExposureBuffer and fwhmExposureGrid parameters are set.
205 """
206 noiseLevel = 1.
207 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6)
208 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7,
209 templateBorderSize=20, doApplyCalibration=True)
211 schema = afwTable.ExposureTable.makeMinimalSchema()
212 weightKey = schema.addField("weight", type="D", doc="Coadd weight")
213 exposureCatalog = afwTable.ExposureCatalog(schema)
214 kernel = measAlg.DoubleGaussianPsf(7, 7, 2.0).getKernel()
215 psf = measAlg.KernelPsf(kernel, template.getBBox().getCenter())
217 record = exposureCatalog.addNew()
218 record.setPsf(psf)
219 record.setWcs(template.wcs)
220 record.setD(weightKey, 1.0)
221 record.setBBox(template.getBBox())
223 customPsf = CustomCoaddPsf(exposureCatalog, template.wcs)
224 template.setPsf(customPsf)
226 # Test that we get an exception if we simply get the FWHM at center.
227 with self.assertRaises(InvalidParameterError):
228 getPsfFwhm(template.psf, True)
230 with self.assertRaises(InvalidParameterError):
231 getPsfFwhm(template.psf, False)
233 # Test that evaluateMeanPsfFwhm runs successfully on the template.
234 evaluateMeanPsfFwhm(template, fwhmExposureBuffer=0.05, fwhmExposureGrid=10)
236 # Since the PSF is spatially invariant, the FWHM should be the same at
237 # all points in the science image.
238 fwhm1 = getPsfFwhm(science.psf, False)
239 fwhm2 = evaluateMeanPsfFwhm(science, fwhmExposureBuffer=0.05, fwhmExposureGrid=10)
240 self.assertAlmostEqual(fwhm1[0], fwhm2, places=13)
241 self.assertAlmostEqual(fwhm1[1], fwhm2, places=13)
243 self.assertAlmostEqual(evaluateMeanPsfFwhm(science, fwhmExposureBuffer=0.05,
244 fwhmExposureGrid=10),
245 getPsfFwhm(science.psf, True), places=7
246 )
248 # Test that the image subtraction task runs successfully.
249 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
250 config.doSubtractBackground = False
251 task = subtractImages.AlardLuptonSubtractTask(config=config)
253 # Test that the task runs if we take the mean FWHM on a grid.
254 with self.assertLogs(level="INFO") as cm:
255 task.run(template, science, sources)
257 # Check that evaluateMeanPsfFwhm was called.
258 # This tests that getPsfFwhm failed raising InvalidParameterError,
259 # that is caught and handled appropriately.
260 logMessage = ("INFO:lsst.alardLuptonSubtract:Unable to evaluate PSF at the average position. "
261 "Evaluting PSF on a grid of points."
262 )
263 self.assertIn(logMessage, cm.output)
265 def test_auto_convolveTemplate(self):
266 """Test that auto mode gives the same result as convolveTemplate when
267 the template psf is the smaller.
268 """
269 noiseLevel = 1.
270 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6)
271 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
272 templateBorderSize=20, doApplyCalibration=True)
273 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
274 config.doSubtractBackground = False
275 config.mode = "convolveTemplate"
277 task = subtractImages.AlardLuptonSubtractTask(config=config)
278 output = task.run(template.clone(), science.clone(), sources)
280 config.mode = "auto"
281 task = subtractImages.AlardLuptonSubtractTask(config=config)
282 outputAuto = task.run(template, science, sources)
283 self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage)
285 def test_auto_convolveScience(self):
286 """Test that auto mode gives the same result as convolveScience when
287 the science psf is the smaller.
288 """
289 noiseLevel = 1.
290 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6)
291 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7,
292 templateBorderSize=20, doApplyCalibration=True)
293 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
294 config.doSubtractBackground = False
295 config.mode = "convolveScience"
297 task = subtractImages.AlardLuptonSubtractTask(config=config)
298 output = task.run(template.clone(), science.clone(), sources)
300 config.mode = "auto"
301 task = subtractImages.AlardLuptonSubtractTask(config=config)
302 outputAuto = task.run(template, science, sources)
303 self.assertMaskedImagesEqual(output.difference.maskedImage, outputAuto.difference.maskedImage)
305 def test_science_better(self):
306 """Test that running with enough sources produces reasonable output,
307 with the science psf being smaller than the template.
308 """
309 statsCtrl = makeStats()
310 statsCtrlDetect = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
312 def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templateNoiseLevel):
313 science, sources = makeTestImage(psfSize=2.0, noiseLevel=scienceNoiseLevel, noiseSeed=6)
314 template, _ = makeTestImage(psfSize=3.0, noiseLevel=templateNoiseLevel, noiseSeed=7,
315 templateBorderSize=20, doApplyCalibration=True)
316 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
317 config.doSubtractBackground = False
318 config.mode = "convolveScience"
319 task = subtractImages.AlardLuptonSubtractTask(config=config)
320 output = task.run(template, science, sources)
321 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
322 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
323 # Mean of difference image should be close to zero.
324 nGoodPix = np.sum(np.isfinite(output.difference.image.array))
325 meanError = (scienceNoiseLevel + templateNoiseLevel)/np.sqrt(nGoodPix)
326 diffimMean = computeRobustStatistics(output.difference.image, output.difference.mask,
327 statsCtrlDetect)
329 self.assertFloatsAlmostEqual(diffimMean, 0, atol=5*meanError)
330 # stddev of difference image should be close to expected value.
331 noiseLevel = np.sqrt(scienceNoiseLevel**2 + templateNoiseLevel**2)
332 varianceMean = computeRobustStatistics(output.difference.variance, output.difference.mask,
333 statsCtrl)
334 diffimStd = computeRobustStatistics(output.difference.image, output.difference.mask,
335 statsCtrl, statistic=afwMath.STDEV)
336 self.assertFloatsAlmostEqual(varianceMean, noiseLevel**2, rtol=0.1)
337 self.assertFloatsAlmostEqual(diffimStd, noiseLevel, rtol=0.1)
339 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=1.)
340 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=.1)
341 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=.1, templateNoiseLevel=.1)
343 def test_template_better(self):
344 """Test that running with enough sources produces reasonable output,
345 with the template psf being smaller than the science.
346 """
347 statsCtrl = makeStats()
348 statsCtrlDetect = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
350 def _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel, templateNoiseLevel):
351 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6)
352 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7,
353 templateBorderSize=20, doApplyCalibration=True)
354 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
355 config.doSubtractBackground = False
356 task = subtractImages.AlardLuptonSubtractTask(config=config)
357 output = task.run(template, science, sources)
358 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"], 1., atol=.05)
359 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"], 1., atol=.05)
360 # There should be no NaNs in the image if we convolve the template with a buffer
361 self.assertTrue(np.all(np.isfinite(output.difference.image.array)))
362 # Mean of difference image should be close to zero.
363 meanError = (scienceNoiseLevel + templateNoiseLevel)/np.sqrt(output.difference.image.array.size)
365 diffimMean = computeRobustStatistics(output.difference.image, output.difference.mask,
366 statsCtrlDetect)
367 self.assertFloatsAlmostEqual(diffimMean, 0, atol=5*meanError)
368 # stddev of difference image should be close to expected value.
369 noiseLevel = np.sqrt(scienceNoiseLevel**2 + templateNoiseLevel**2)
370 varianceMean = computeRobustStatistics(output.difference.variance, output.difference.mask,
371 statsCtrl)
372 diffimStd = computeRobustStatistics(output.difference.image, output.difference.mask,
373 statsCtrl, statistic=afwMath.STDEV)
374 self.assertFloatsAlmostEqual(varianceMean, noiseLevel**2, rtol=0.1)
375 self.assertFloatsAlmostEqual(diffimStd, noiseLevel, rtol=0.1)
377 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=1.)
378 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=1., templateNoiseLevel=.1)
379 _run_and_check_images(statsCtrl, statsCtrlDetect, scienceNoiseLevel=.1, templateNoiseLevel=.1)
381 def test_symmetry(self):
382 """Test that convolving the science and convolving the template are
383 symmetric: if the psfs are switched between them, the difference image
384 should be nearly the same.
385 """
386 noiseLevel = 1.
387 # Don't include a border for the template, in order to make the results
388 # comparable when we swap which image is treated as the "science" image.
389 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel,
390 noiseSeed=6, templateBorderSize=0, doApplyCalibration=True)
391 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel,
392 noiseSeed=7, templateBorderSize=0, doApplyCalibration=True)
393 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
394 config.mode = 'auto'
395 config.doSubtractBackground = False
396 task = subtractImages.AlardLuptonSubtractTask(config=config)
398 # The science image will be modified in place, so use a copy for the second run.
399 science_better = task.run(template.clone(), science.clone(), sources)
400 template_better = task.run(science, template, sources)
402 delta = template_better.difference.clone()
403 delta.image -= science_better.difference.image
404 delta.variance -= science_better.difference.variance
405 delta.mask.array -= science_better.difference.mask.array
407 statsCtrl = makeStats()
408 # Mean of delta should be very close to zero.
409 nGoodPix = np.sum(np.isfinite(delta.image.array))
410 meanError = 2*noiseLevel/np.sqrt(nGoodPix)
411 deltaMean = computeRobustStatistics(delta.image, delta.mask, statsCtrl)
412 deltaStd = computeRobustStatistics(delta.image, delta.mask, statsCtrl, statistic=afwMath.STDEV)
413 self.assertFloatsAlmostEqual(deltaMean, 0, atol=5*meanError)
414 # stddev of difference image should be close to expected value
415 self.assertFloatsAlmostEqual(deltaStd, 2*np.sqrt(2)*noiseLevel, rtol=.1)
417 def test_few_sources(self):
418 """Test with only 1 source, to check that we get a useful error.
419 """
420 xSize = 256
421 ySize = 256
422 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize)
423 template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True)
424 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
425 task = subtractImages.AlardLuptonSubtractTask(config=config)
426 sources = sources[0:1]
427 with self.assertRaisesRegex(RuntimeError,
428 "Cannot compute PSF matching kernel: too few sources selected."):
429 task.run(template, science, sources)
431 def test_kernel_source_selector(self):
432 """Check that kernel source selection behaves as expected.
433 """
434 xSize = 256
435 ySize = 256
436 nSourcesSimulated = 20
437 science, sources = makeTestImage(psfSize=2.4, nSrc=nSourcesSimulated,
438 xSize=xSize, ySize=ySize)
439 template, _ = makeTestImage(psfSize=2.0, nSrc=nSourcesSimulated,
440 xSize=xSize, ySize=ySize, doApplyCalibration=True)
441 badSourceFlag = "slot_Centroid_flag"
443 def _run_and_check_sources(sourcesIn, maxKernelSources=1000, minKernelSources=3):
444 sources = sourcesIn.copy(deep=True)
445 # Verify that source flags are not set in the input catalog
446 self.assertEqual(np.sum(sources[badSourceFlag]), 0)
447 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
448 config.badSourceFlags = [badSourceFlag, ]
449 config.maxKernelSources = maxKernelSources
450 config.minKernelSources = minKernelSources
452 task = subtractImages.AlardLuptonSubtractTask(config=config)
453 nSources = len(sources)
454 # Flag a third of the sources
455 sources[0:: 3][badSourceFlag] = True
456 nBadSources = np.sum(sources[badSourceFlag])
457 if maxKernelSources > 0:
458 nGoodSources = np.minimum(nSources - nBadSources, maxKernelSources)
459 else:
460 nGoodSources = nSources - nBadSources
462 signalToNoise = sources.getPsfInstFlux()/sources.getPsfInstFluxErr()
463 signalToNoise = signalToNoise[~sources[badSourceFlag]]
464 signalToNoise.sort()
465 selectSources = task._sourceSelector(sources, science.mask)
466 self.assertEqual(nGoodSources, len(selectSources))
467 signalToNoiseOut = selectSources.getPsfInstFlux()/selectSources.getPsfInstFluxErr()
468 signalToNoiseOut.sort()
469 self.assertFloatsAlmostEqual(signalToNoise[-nGoodSources:], signalToNoiseOut)
471 _run_and_check_sources(sources)
472 _run_and_check_sources(sources, maxKernelSources=len(sources)//3)
473 _run_and_check_sources(sources, maxKernelSources=-1)
474 with self.assertRaises(RuntimeError):
475 _run_and_check_sources(sources, minKernelSources=1000)
477 def test_order_equal_images(self):
478 """Verify that the result is the same regardless of convolution mode
479 if the images are equivalent.
480 """
481 noiseLevel = .1
482 seed1 = 6
483 seed2 = 7
484 science1, sources1 = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed1,
485 clearEdgeMask=True)
486 template1, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2,
487 templateBorderSize=0, doApplyCalibration=True,
488 clearEdgeMask=True)
489 config1 = subtractImages.AlardLuptonSubtractTask.ConfigClass()
490 config1.mode = "convolveTemplate"
491 config1.doSubtractBackground = False
492 task1 = subtractImages.AlardLuptonSubtractTask(config=config1)
493 results_convolveTemplate = task1.run(template1, science1, sources1)
495 science2, sources2 = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed1,
496 clearEdgeMask=True)
497 template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=seed2,
498 templateBorderSize=0, doApplyCalibration=True,
499 clearEdgeMask=True)
500 config2 = subtractImages.AlardLuptonSubtractTask.ConfigClass()
501 config2.mode = "convolveScience"
502 config2.doSubtractBackground = False
503 task2 = subtractImages.AlardLuptonSubtractTask(config=config2)
504 results_convolveScience = task2.run(template2, science2, sources2)
505 bbox = results_convolveTemplate.difference.getBBox().clippedTo(
506 results_convolveScience.difference.getBBox())
507 diff1 = science1.maskedImage.clone()[bbox]
508 diff1 -= template1.maskedImage[bbox]
509 diff2 = science2.maskedImage.clone()[bbox]
510 diff2 -= template2.maskedImage[bbox]
511 self.assertFloatsAlmostEqual(results_convolveTemplate.difference[bbox].image.array,
512 diff1.image.array,
513 atol=noiseLevel*5.)
514 self.assertFloatsAlmostEqual(results_convolveScience.difference[bbox].image.array,
515 diff2.image.array,
516 atol=noiseLevel*5.)
517 diffErr = noiseLevel*2
518 self.assertMaskedImagesAlmostEqual(results_convolveTemplate.difference[bbox].maskedImage,
519 results_convolveScience.difference[bbox].maskedImage,
520 atol=diffErr*5.)
522 def test_background_subtraction(self):
523 """Check that we can recover the background,
524 and that it is subtracted correctly in the difference image.
525 """
526 noiseLevel = 1.
527 xSize = 512
528 ySize = 512
529 x0 = 123
530 y0 = 456
531 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
532 templateBorderSize=20,
533 xSize=xSize, ySize=ySize, x0=x0, y0=y0,
534 doApplyCalibration=True)
535 params = [2.2, 2.1, 2.0, 1.2, 1.1, 1.0]
537 bbox2D = lsst.geom.Box2D(lsst.geom.Point2D(x0, y0), lsst.geom.Extent2D(xSize, ySize))
538 background_model = afwMath.Chebyshev1Function2D(params, bbox2D)
539 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6,
540 background=background_model,
541 xSize=xSize, ySize=ySize, x0=x0, y0=y0)
542 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
543 config.doSubtractBackground = True
545 config.makeKernel.kernel.name = "AL"
546 config.makeKernel.kernel.active.fitForBackground = True
547 config.makeKernel.kernel.active.spatialKernelOrder = 1
548 config.makeKernel.kernel.active.spatialBgOrder = 2
549 statsCtrl = makeStats()
551 def _run_and_check_images(config, statsCtrl, mode):
552 """Check that the fit background matches the input model.
553 """
554 config.mode = mode
555 task = subtractImages.AlardLuptonSubtractTask(config=config)
556 output = task.run(template.clone(), science.clone(), sources)
558 # We should be fitting the same number of parameters as were in the input model
559 self.assertEqual(output.backgroundModel.getNParameters(), background_model.getNParameters())
561 # The parameters of the background fit should be close to the input model
562 self.assertFloatsAlmostEqual(np.array(output.backgroundModel.getParameters()),
563 np.array(params), rtol=0.3)
565 # stddev of difference image should be close to expected value.
566 # This will fail if we have mis-subtracted the background.
567 stdVal = computeRobustStatistics(output.difference.image, output.difference.mask,
568 statsCtrl, statistic=afwMath.STDEV)
569 self.assertFloatsAlmostEqual(stdVal, np.sqrt(2)*noiseLevel, rtol=0.1)
571 _run_and_check_images(config, statsCtrl, "convolveTemplate")
572 _run_and_check_images(config, statsCtrl, "convolveScience")
574 def test_scale_variance_convolve_template(self):
575 """Check variance scaling of the image difference.
576 """
577 scienceNoiseLevel = 4.
578 templateNoiseLevel = 2.
579 scaleFactor = 1.345
580 # Make sure to include pixels with the DETECTED mask bit set.
581 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
583 def _run_and_check_images(science, template, sources, statsCtrl,
584 doDecorrelation, doScaleVariance, scaleFactor=1.):
585 """Check that the variance plane matches the expected value for
586 different configurations of ``doDecorrelation`` and ``doScaleVariance``.
587 """
589 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
590 config.doSubtractBackground = False
591 config.doDecorrelation = doDecorrelation
592 config.doScaleVariance = doScaleVariance
593 task = subtractImages.AlardLuptonSubtractTask(config=config)
594 output = task.run(template.clone(), science.clone(), sources)
595 if doScaleVariance:
596 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
597 scaleFactor, atol=0.05)
598 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
599 scaleFactor, atol=0.05)
601 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl)
602 if doDecorrelation:
603 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl)
604 else:
605 templateNoise = computeRobustStatistics(output.matchedTemplate.variance,
606 output.matchedTemplate.mask,
607 statsCtrl)
609 if doScaleVariance:
610 templateNoise *= scaleFactor
611 scienceNoise *= scaleFactor
612 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
613 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
615 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6)
616 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7,
617 templateBorderSize=20, doApplyCalibration=True)
618 # Verify that the variance plane of the difference image is correct
619 # when the template and science variance planes are correct
620 _run_and_check_images(science, template, sources, statsCtrl,
621 doDecorrelation=True, doScaleVariance=True)
622 _run_and_check_images(science, template, sources, statsCtrl,
623 doDecorrelation=True, doScaleVariance=False)
624 _run_and_check_images(science, template, sources, statsCtrl,
625 doDecorrelation=False, doScaleVariance=True)
626 _run_and_check_images(science, template, sources, statsCtrl,
627 doDecorrelation=False, doScaleVariance=False)
629 # Verify that the variance plane of the difference image is correct
630 # when the template variance plane is incorrect
631 template.variance.array /= scaleFactor
632 science.variance.array /= scaleFactor
633 _run_and_check_images(science, template, sources, statsCtrl,
634 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
635 _run_and_check_images(science, template, sources, statsCtrl,
636 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor)
637 _run_and_check_images(science, template, sources, statsCtrl,
638 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor)
639 _run_and_check_images(science, template, sources, statsCtrl,
640 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor)
642 def test_scale_variance_convolve_science(self):
643 """Check variance scaling of the image difference.
644 """
645 scienceNoiseLevel = 4.
646 templateNoiseLevel = 2.
647 scaleFactor = 1.345
648 # Make sure to include pixels with the DETECTED mask bit set.
649 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
651 def _run_and_check_images(science, template, sources, statsCtrl,
652 doDecorrelation, doScaleVariance, scaleFactor=1.):
653 """Check that the variance plane matches the expected value for
654 different configurations of ``doDecorrelation`` and ``doScaleVariance``.
655 """
657 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
658 config.mode = "convolveScience"
659 config.doSubtractBackground = False
660 config.doDecorrelation = doDecorrelation
661 config.doScaleVariance = doScaleVariance
662 task = subtractImages.AlardLuptonSubtractTask(config=config)
663 output = task.run(template.clone(), science.clone(), sources)
664 if doScaleVariance:
665 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
666 scaleFactor, atol=0.05)
667 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
668 scaleFactor, atol=0.05)
670 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl)
671 if doDecorrelation:
672 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl)
673 else:
674 scienceNoise = computeRobustStatistics(output.matchedScience.variance,
675 output.matchedScience.mask,
676 statsCtrl)
678 if doScaleVariance:
679 templateNoise *= scaleFactor
680 scienceNoise *= scaleFactor
682 varMean = computeRobustStatistics(output.difference.variance, output.difference.mask, statsCtrl)
683 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
685 science, sources = makeTestImage(psfSize=2.0, noiseLevel=scienceNoiseLevel, noiseSeed=6)
686 template, _ = makeTestImage(psfSize=3.0, noiseLevel=templateNoiseLevel, noiseSeed=7,
687 templateBorderSize=20, doApplyCalibration=True)
688 # Verify that the variance plane of the difference image is correct
689 # when the template and science variance planes are correct
690 _run_and_check_images(science, template, sources, statsCtrl,
691 doDecorrelation=True, doScaleVariance=True)
692 _run_and_check_images(science, template, sources, statsCtrl,
693 doDecorrelation=True, doScaleVariance=False)
694 _run_and_check_images(science, template, sources, statsCtrl,
695 doDecorrelation=False, doScaleVariance=True)
696 _run_and_check_images(science, template, sources, statsCtrl,
697 doDecorrelation=False, doScaleVariance=False)
699 # Verify that the variance plane of the difference image is correct
700 # when the template and science variance planes are incorrect
701 science.variance.array /= scaleFactor
702 template.variance.array /= scaleFactor
703 _run_and_check_images(science, template, sources, statsCtrl,
704 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
705 _run_and_check_images(science, template, sources, statsCtrl,
706 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor)
707 _run_and_check_images(science, template, sources, statsCtrl,
708 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor)
709 _run_and_check_images(science, template, sources, statsCtrl,
710 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor)
712 def test_exposure_properties_convolve_template(self):
713 """Check that all necessary exposure metadata is included
714 when the template is convolved.
715 """
716 noiseLevel = 1.
717 seed = 37
718 rng = np.random.RandomState(seed)
719 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6)
720 psf = science.psf
721 psfAvgPos = psf.getAveragePosition()
722 psfSize = getPsfFwhm(science.psf)
723 psfImg = psf.computeKernelImage(psfAvgPos)
724 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
725 templateBorderSize=20, doApplyCalibration=True)
727 # Generate a random aperture correction map
728 apCorrMap = lsst.afw.image.ApCorrMap()
729 for name in ("a", "b", "c"):
730 apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3)))
731 science.info.setApCorrMap(apCorrMap)
733 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
734 config.mode = "convolveTemplate"
736 def _run_and_check_images(doDecorrelation):
737 """Check that the metadata is correct with or without decorrelation.
738 """
739 config.doDecorrelation = doDecorrelation
740 task = subtractImages.AlardLuptonSubtractTask(config=config)
741 output = task.run(template.clone(), science.clone(), sources)
742 psfOut = output.difference.psf
743 psfAvgPos = psfOut.getAveragePosition()
744 if doDecorrelation:
745 # Decorrelation requires recalculating the PSF,
746 # so it will not be the same as the input
747 psfOutSize = getPsfFwhm(science.psf)
748 self.assertFloatsAlmostEqual(psfSize, psfOutSize)
749 else:
750 psfOutImg = psfOut.computeKernelImage(psfAvgPos)
751 self.assertImagesAlmostEqual(psfImg, psfOutImg)
753 # check PSF, WCS, bbox, filterLabel, photoCalib, aperture correction
754 self._compare_apCorrMaps(apCorrMap, output.difference.info.getApCorrMap())
755 self.assertWcsAlmostEqualOverBBox(science.wcs, output.difference.wcs, science.getBBox())
756 self.assertEqual(science.filter, output.difference.filter)
757 self.assertEqual(science.photoCalib, output.difference.photoCalib)
758 _run_and_check_images(doDecorrelation=True)
759 _run_and_check_images(doDecorrelation=False)
761 def test_exposure_properties_convolve_science(self):
762 """Check that all necessary exposure metadata is included
763 when the science image is convolved.
764 """
765 noiseLevel = 1.
766 seed = 37
767 rng = np.random.RandomState(seed)
768 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6)
769 template, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=7,
770 templateBorderSize=20, doApplyCalibration=True)
771 psf = template.psf
772 psfAvgPos = psf.getAveragePosition()
773 psfSize = getPsfFwhm(template.psf)
774 psfImg = psf.computeKernelImage(psfAvgPos)
776 # Generate a random aperture correction map
777 apCorrMap = lsst.afw.image.ApCorrMap()
778 for name in ("a", "b", "c"):
779 apCorrMap.set(name, lsst.afw.math.ChebyshevBoundedField(science.getBBox(), rng.randn(3, 3)))
780 science.info.setApCorrMap(apCorrMap)
782 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
783 config.mode = "convolveScience"
785 def _run_and_check_images(doDecorrelation):
786 """Check that the metadata is correct with or without decorrelation.
787 """
788 config.doDecorrelation = doDecorrelation
789 task = subtractImages.AlardLuptonSubtractTask(config=config)
790 output = task.run(template.clone(), science.clone(), sources)
791 if doDecorrelation:
792 # Decorrelation requires recalculating the PSF,
793 # so it will not be the same as the input
794 psfOutSize = getPsfFwhm(template.psf)
795 self.assertFloatsAlmostEqual(psfSize, psfOutSize)
796 else:
797 psfOut = output.difference.psf
798 psfAvgPos = psfOut.getAveragePosition()
799 psfOutImg = psfOut.computeKernelImage(psfAvgPos)
800 self.assertImagesAlmostEqual(psfImg, psfOutImg)
802 # check PSF, WCS, bbox, filterLabel, photoCalib, aperture correction
803 self._compare_apCorrMaps(apCorrMap, output.difference.info.getApCorrMap())
804 self.assertWcsAlmostEqualOverBBox(science.wcs, output.difference.wcs, science.getBBox())
805 self.assertEqual(science.filter, output.difference.filter)
806 self.assertEqual(science.photoCalib, output.difference.photoCalib)
808 _run_and_check_images(doDecorrelation=True)
809 _run_and_check_images(doDecorrelation=False)
811 def _compare_apCorrMaps(self, a, b):
812 """Compare two ApCorrMaps for equality, without assuming that their BoundedFields have the
813 same addresses (i.e. so we can compare after serialization).
815 This function is taken from ``ApCorrMapTestCase`` in afw/tests/.
817 Parameters
818 ----------
819 a, b : `lsst.afw.image.ApCorrMap`
820 The two aperture correction maps to compare.
821 """
822 self.assertEqual(len(a), len(b))
823 for name, value in list(a.items()):
824 value2 = b.get(name)
825 self.assertIsNotNone(value2)
826 self.assertEqual(value.getBBox(), value2.getBBox())
827 self.assertFloatsAlmostEqual(
828 value.getCoefficients(), value2.getCoefficients(), rtol=0.0)
830 def test_fake_mask_plane_propagation(self):
831 """Test that we have the mask planes related to fakes in diffim images.
832 This is testing method called updateMasks
833 """
834 xSize = 200
835 ySize = 200
836 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize)
837 science_fake_img, science_fake_sources = makeTestImage(
838 psfSize=2.4, xSize=xSize, ySize=ySize, seed=7, nSrc=2, noiseLevel=0.25, fluxRange=1
839 )
840 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
841 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
842 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=2, noiseLevel=0.25, fluxRange=1
843 )
844 # created fakes and added them to the images
845 science.image.array += science_fake_img.image.array
846 template.image.array += tmplt_fake_img.image.array
848 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
849 # adding mask planes to both science and template images
850 science_mask_planes = science.mask.addMaskPlane("FAKE")
851 template_mask_planes = template.mask.addMaskPlane("FAKE")
853 for a_science_source in science_fake_sources:
854 # 3 x 3 masking of the source locations is fine
855 bbox = lsst.geom.Box2I(
856 lsst.geom.Point2I(a_science_source.getX(), a_science_source.getY()), lsst.geom.Extent2I(3, 3)
857 )
858 science[bbox].mask.array |= science_mask_planes
860 for a_template_source in tmplt_fake_sources:
861 # 3 x 3 masking of the source locations is fine
862 bbox = lsst.geom.Box2I(
863 lsst.geom.Point2I(a_template_source.getX(), a_template_source.getY()),
864 lsst.geom.Extent2I(3, 3)
865 )
866 template[bbox].mask.array |= template_mask_planes
868 science_fake_masked = (science.mask.array & science.mask.getPlaneBitMask("FAKE")) > 0
869 template_fake_masked = (template.mask.array & template.mask.getPlaneBitMask("FAKE")) > 0
871 config = subtractImages.AlardLuptonSubtractTask.ConfigClass()
872 task = subtractImages.AlardLuptonSubtractTask(config=config)
873 subtraction = task.run(template, science, sources)
875 # check subtraction mask plane is set where we set the previous masks
876 diff_mask = subtraction.difference.mask
878 # science mask should be now in INJECTED
879 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
881 # template mask should be now in INJECTED_TEMPLATE
882 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
884 self.assertEqual(np.sum(inj_masked.astype(int)-science_fake_masked.astype(int)), 0)
885 self.assertEqual(np.sum(injTmplt_masked.astype(int)-template_fake_masked.astype(int)), 0)
888class AlardLuptonPreconvolveSubtractTest(lsst.utils.tests.TestCase):
890 def test_mismatched_template(self):
891 """Test that an error is raised if the template
892 does not fully contain the science image.
893 """
894 xSize = 200
895 ySize = 200
896 science, sources = makeTestImage(psfSize=2.4, xSize=xSize + 20, ySize=ySize + 20)
897 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
898 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
899 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
900 with self.assertRaises(AssertionError):
901 task.run(template, science, sources)
903 def test_equal_images(self):
904 """Test that running with enough sources produces reasonable output,
905 with the same size psf in the template and science.
906 """
907 noiseLevel = 1.
908 xSize = 400
909 ySize = 400
910 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=6,
911 xSize=xSize, ySize=ySize)
912 template, _ = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel, noiseSeed=7,
913 templateBorderSize=20, doApplyCalibration=True,
914 xSize=xSize, ySize=ySize)
915 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
916 config.doSubtractBackground = False
917 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
918 output = task.run(template, science, sources)
919 # There shoud be no NaN values in the Score image
920 self.assertTrue(np.all(np.isfinite(output.scoreExposure.image.array)))
921 # Mean of Score image should be close to zero.
922 meanError = noiseLevel/np.sqrt(output.scoreExposure.image.array.size)
923 # Make sure to include pixels with the DETECTED mask bit set.
924 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
925 scoreMean = computeRobustStatistics(output.scoreExposure.image,
926 output.scoreExposure.mask,
927 statsCtrl)
928 self.assertFloatsAlmostEqual(scoreMean, 0, atol=5*meanError)
929 nea = computePSFNoiseEquivalentArea(science.psf)
930 # stddev of Score image should be close to expected value.
931 scoreStd = computeRobustStatistics(output.scoreExposure.image, output.scoreExposure.mask,
932 statsCtrl=statsCtrl, statistic=afwMath.STDEV)
933 self.assertFloatsAlmostEqual(scoreStd, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=0.1)
935 def test_incomplete_template_coverage(self):
936 noiseLevel = 1.
937 border = 20
938 xSize = 400
939 ySize = 400
940 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6,
941 xSize=xSize, ySize=ySize)
942 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
943 templateBorderSize=border, doApplyCalibration=True,
944 xSize=xSize, ySize=ySize)
946 science_height = science.getBBox().getDimensions().getY()
948 def _run_and_check_coverage(template_coverage):
949 template_cut = template.clone()
950 template_height = int(science_height*template_coverage + border)
951 template_cut.image.array[:, template_height:] = 0
952 template_cut.mask.array[:, template_height:] = template_cut.mask.getPlaneBitMask('NO_DATA')
953 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
954 if template_coverage < config.requiredTemplateFraction:
955 doRaise = True
956 elif template_coverage < config.minTemplateFractionForExpectedSuccess:
957 doRaise = True
958 else:
959 doRaise = False
960 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
961 if doRaise:
962 with self.assertRaises(NoWorkFound):
963 task.run(template_cut, science.clone(), sources.copy(deep=True))
964 else:
965 task.run(template_cut, science.clone(), sources.copy(deep=True))
966 _run_and_check_coverage(template_coverage=0.09)
967 _run_and_check_coverage(template_coverage=0.19)
968 _run_and_check_coverage(template_coverage=.7)
970 def test_clear_template_mask(self):
971 noiseLevel = 1.
972 xSize = 400
973 ySize = 400
974 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6,
975 xSize=xSize, ySize=ySize)
976 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
977 templateBorderSize=20, doApplyCalibration=True,
978 xSize=xSize, ySize=ySize)
979 diffimEmptyMaskPlanes = ["DETECTED", "DETECTED_NEGATIVE"]
980 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
981 config.doSubtractBackground = False # Ensure that each each mask plane is set for some pixels
982 mask = template.mask
983 x0 = 50
984 x1 = 75
985 y0 = 150
986 y1 = 175
987 scienceMaskCheck = {}
988 for maskPlane in mask.getMaskPlaneDict().keys():
989 scienceMaskCheck[maskPlane] = np.sum(science.mask.array & mask.getPlaneBitMask(maskPlane) > 0)
990 mask.array[x0: x1, y0: y1] |= mask.getPlaneBitMask(maskPlane)
991 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0))
993 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
994 output = task.run(template, science, sources)
995 # Verify that the template mask has been modified in place
996 for maskPlane in mask.getMaskPlaneDict().keys():
997 if maskPlane in diffimEmptyMaskPlanes:
998 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0))
999 elif maskPlane in config.preserveTemplateMask:
1000 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) > 0))
1001 else:
1002 self.assertTrue(np.sum(mask.array & mask.getPlaneBitMask(maskPlane) == 0))
1003 # Mask planes set in the science image should also be set in the difference
1004 # Except the "DETECTED" planes should have been cleared
1005 diffimMask = output.scoreExposure.mask
1006 for maskPlane, scienceSum in scienceMaskCheck.items():
1007 diffimSum = np.sum(diffimMask.array & mask.getPlaneBitMask(maskPlane) > 0)
1008 if maskPlane in diffimEmptyMaskPlanes:
1009 self.assertEqual(diffimSum, 0)
1010 else:
1011 self.assertTrue(diffimSum >= scienceSum)
1013 def test_agnostic_template_psf(self):
1014 """Test that the Score image is the same whether the template PSF is
1015 larger or smaller than the science image PSF.
1016 """
1017 noiseLevel = .3
1018 xSize = 400
1019 ySize = 400
1020 science, sources = makeTestImage(psfSize=2.4, noiseLevel=noiseLevel,
1021 noiseSeed=6, templateBorderSize=0,
1022 xSize=xSize, ySize=ySize)
1023 template1, _ = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel,
1024 noiseSeed=7, doApplyCalibration=True,
1025 xSize=xSize, ySize=ySize)
1026 template2, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel,
1027 noiseSeed=8, doApplyCalibration=True,
1028 xSize=xSize, ySize=ySize)
1029 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
1030 config.doSubtractBackground = False
1031 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
1033 science_better = task.run(template1, science.clone(), sources)
1034 template_better = task.run(template2, science, sources)
1035 bbox = science_better.scoreExposure.getBBox().clippedTo(template_better.scoreExposure.getBBox())
1037 delta = template_better.scoreExposure[bbox].clone()
1038 delta.image -= science_better.scoreExposure[bbox].image
1039 delta.variance -= science_better.scoreExposure[bbox].variance
1040 delta.mask.array &= science_better.scoreExposure[bbox].mask.array
1042 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
1043 # Mean of delta should be very close to zero.
1044 nGoodPix = np.sum(np.isfinite(delta.image.array))
1045 meanError = 2*noiseLevel/np.sqrt(nGoodPix)
1046 deltaMean = computeRobustStatistics(delta.image, delta.mask, statsCtrl)
1047 deltaStd = computeRobustStatistics(delta.image, delta.mask, statsCtrl,
1048 statistic=afwMath.STDEV)
1049 self.assertFloatsAlmostEqual(deltaMean, 0, atol=5*meanError)
1050 nea = computePSFNoiseEquivalentArea(science.psf)
1051 # stddev of Score image should be close to expected value
1052 self.assertFloatsAlmostEqual(deltaStd, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=.1)
1054 def test_few_sources(self):
1055 """Test with only 1 source, to check that we get a useful error.
1056 """
1057 xSize = 256
1058 ySize = 256
1059 science, sources = makeTestImage(psfSize=2.4, nSrc=10, xSize=xSize, ySize=ySize)
1060 template, _ = makeTestImage(psfSize=2.0, nSrc=10, xSize=xSize, ySize=ySize, doApplyCalibration=True)
1061 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
1062 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
1063 sources = sources[0:1]
1064 with self.assertRaisesRegex(RuntimeError,
1065 "Cannot compute PSF matching kernel: too few sources selected."):
1066 task.run(template, science, sources)
1068 def test_background_subtraction(self):
1069 """Check that we can recover the background,
1070 and that it is subtracted correctly in the Score image.
1071 """
1072 noiseLevel = 1.
1073 xSize = 512
1074 ySize = 512
1075 x0 = 123
1076 y0 = 456
1077 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
1078 templateBorderSize=20,
1079 xSize=xSize, ySize=ySize, x0=x0, y0=y0,
1080 doApplyCalibration=True)
1081 params = [2.2, 2.1, 2.0, 1.2, 1.1, 1.0]
1083 bbox2D = lsst.geom.Box2D(lsst.geom.Point2D(x0, y0), lsst.geom.Extent2D(xSize, ySize))
1084 background_model = afwMath.Chebyshev1Function2D(params, bbox2D)
1085 science, sources = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=6,
1086 background=background_model,
1087 xSize=xSize, ySize=ySize, x0=x0, y0=y0)
1088 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
1089 config.doSubtractBackground = True
1091 config.makeKernel.kernel.name = "AL"
1092 config.makeKernel.kernel.active.fitForBackground = True
1093 config.makeKernel.kernel.active.spatialKernelOrder = 1
1094 config.makeKernel.kernel.active.spatialBgOrder = 2
1095 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
1097 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
1098 output = task.run(template.clone(), science.clone(), sources)
1100 # We should be fitting the same number of parameters as were in the input model
1101 self.assertEqual(output.backgroundModel.getNParameters(), background_model.getNParameters())
1103 # The parameters of the background fit should be close to the input model
1104 self.assertFloatsAlmostEqual(np.array(output.backgroundModel.getParameters()),
1105 np.array(params), rtol=0.2)
1107 # stddev of Score image should be close to expected value.
1108 # This will fail if we have mis-subtracted the background.
1109 stdVal = computeRobustStatistics(output.scoreExposure.image, output.scoreExposure.mask,
1110 statsCtrl, statistic=afwMath.STDEV)
1111 # get the img psf Noise Equivalent Area value
1112 nea = computePSFNoiseEquivalentArea(science.psf)
1113 self.assertFloatsAlmostEqual(stdVal, np.sqrt(2)*noiseLevel/np.sqrt(nea), rtol=0.1)
1115 def test_scale_variance(self):
1116 """Check variance scaling of the Score image.
1117 """
1118 scienceNoiseLevel = 4.
1119 templateNoiseLevel = 2.
1120 scaleFactor = 1.345
1121 xSize = 400
1122 ySize = 400
1123 # Make sure to include pixels with the DETECTED mask bit set.
1124 statsCtrl = makeStats(badMaskPlanes=("EDGE", "BAD", "NO_DATA"))
1126 def _run_and_check_images(science, template, sources, statsCtrl,
1127 doDecorrelation, doScaleVariance, scaleFactor=1.):
1128 """Check that the variance plane matches the expected value for
1129 different configurations of ``doDecorrelation`` and ``doScaleVariance``.
1130 """
1132 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
1133 config.doSubtractBackground = False
1134 config.doDecorrelation = doDecorrelation
1135 config.doScaleVariance = doScaleVariance
1136 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
1137 output = task.run(template.clone(), science.clone(), sources)
1138 if doScaleVariance:
1139 self.assertFloatsAlmostEqual(task.metadata["scaleTemplateVarianceFactor"],
1140 scaleFactor, atol=0.05)
1141 self.assertFloatsAlmostEqual(task.metadata["scaleScienceVarianceFactor"],
1142 scaleFactor, atol=0.05)
1144 scienceNoise = computeRobustStatistics(science.variance, science.mask, statsCtrl)
1145 # get the img psf Noise Equivalent Area value
1146 nea = computePSFNoiseEquivalentArea(science.psf)
1147 scienceNoise /= nea
1148 if doDecorrelation:
1149 templateNoise = computeRobustStatistics(template.variance, template.mask, statsCtrl)
1150 templateNoise /= nea
1151 else:
1152 # Don't divide by NEA in this case, since the template is convolved
1153 # and in the same units as the Score exposure.
1154 templateNoise = computeRobustStatistics(output.matchedTemplate.variance,
1155 output.matchedTemplate.mask,
1156 statsCtrl)
1157 if doScaleVariance:
1158 templateNoise *= scaleFactor
1159 scienceNoise *= scaleFactor
1160 varMean = computeRobustStatistics(output.scoreExposure.variance,
1161 output.scoreExposure.mask,
1162 statsCtrl)
1163 self.assertFloatsAlmostEqual(varMean, scienceNoise + templateNoise, rtol=0.1)
1165 science, sources = makeTestImage(psfSize=3.0, noiseLevel=scienceNoiseLevel, noiseSeed=6,
1166 xSize=xSize, ySize=ySize)
1167 template, _ = makeTestImage(psfSize=2.0, noiseLevel=templateNoiseLevel, noiseSeed=7,
1168 templateBorderSize=20, doApplyCalibration=True,
1169 xSize=xSize, ySize=ySize)
1170 # Verify that the variance plane of the Score image is correct
1171 # when the template and science variance planes are correct
1172 _run_and_check_images(science, template, sources, statsCtrl,
1173 doDecorrelation=True, doScaleVariance=True)
1174 _run_and_check_images(science, template, sources, statsCtrl,
1175 doDecorrelation=True, doScaleVariance=False)
1176 _run_and_check_images(science, template, sources, statsCtrl,
1177 doDecorrelation=False, doScaleVariance=True)
1178 _run_and_check_images(science, template, sources, statsCtrl,
1179 doDecorrelation=False, doScaleVariance=False)
1181 # Verify that the variance plane of the Score image is correct
1182 # when the template variance plane is incorrect
1183 template.variance.array /= scaleFactor
1184 science.variance.array /= scaleFactor
1185 _run_and_check_images(science, template, sources, statsCtrl,
1186 doDecorrelation=True, doScaleVariance=True, scaleFactor=scaleFactor)
1187 _run_and_check_images(science, template, sources, statsCtrl,
1188 doDecorrelation=True, doScaleVariance=False, scaleFactor=scaleFactor)
1189 _run_and_check_images(science, template, sources, statsCtrl,
1190 doDecorrelation=False, doScaleVariance=True, scaleFactor=scaleFactor)
1191 _run_and_check_images(science, template, sources, statsCtrl,
1192 doDecorrelation=False, doScaleVariance=False, scaleFactor=scaleFactor)
1194 def test_exposure_properties(self):
1195 """Check that all necessary exposure metadata is included
1196 with the Score image.
1197 """
1198 noiseLevel = 1.
1199 xSize = 400
1200 ySize = 400
1201 science, sources = makeTestImage(psfSize=3.0, noiseLevel=noiseLevel, noiseSeed=6,
1202 xSize=xSize, ySize=ySize)
1203 psf = science.psf
1204 psfAvgPos = psf.getAveragePosition()
1205 psfSize = getPsfFwhm(science.psf)
1206 psfImg = psf.computeKernelImage(psfAvgPos)
1207 template, _ = makeTestImage(psfSize=2.0, noiseLevel=noiseLevel, noiseSeed=7,
1208 templateBorderSize=20, doApplyCalibration=True,
1209 xSize=xSize, ySize=ySize)
1211 config = subtractImages.AlardLuptonPreconvolveSubtractTask.ConfigClass()
1213 def _run_and_check_images(doDecorrelation):
1214 """Check that the metadata is correct with or without decorrelation.
1215 """
1216 config.doDecorrelation = doDecorrelation
1217 task = subtractImages.AlardLuptonPreconvolveSubtractTask(config=config)
1218 output = task.run(template.clone(), science.clone(), sources)
1219 psfOut = output.scoreExposure.psf
1220 psfAvgPos = psfOut.getAveragePosition()
1221 if doDecorrelation:
1222 # Decorrelation requires recalculating the PSF,
1223 # so it will not be the same as the input
1224 psfOutSize = getPsfFwhm(science.psf)
1225 self.assertFloatsAlmostEqual(psfSize, psfOutSize)
1226 else:
1227 psfOutImg = psfOut.computeKernelImage(psfAvgPos)
1228 self.assertImagesAlmostEqual(psfImg, psfOutImg)
1230 # check PSF, WCS, bbox, filterLabel, photoCalib
1231 self.assertWcsAlmostEqualOverBBox(science.wcs, output.scoreExposure.wcs, science.getBBox())
1232 self.assertEqual(science.filter, output.scoreExposure.filter)
1233 self.assertEqual(science.photoCalib, output.scoreExposure.photoCalib)
1234 _run_and_check_images(doDecorrelation=True)
1235 _run_and_check_images(doDecorrelation=False)
1238def setup_module(module):
1239 lsst.utils.tests.init()
1242class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
1243 pass
1246if __name__ == "__main__": 1246 ↛ 1247line 1246 didn't jump to line 1247, because the condition on line 1246 was never true
1247 lsst.utils.tests.init()
1248 unittest.main()