Coverage for tests/test_detectAndMeasure.py: 6%
500 statements
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-14 11:41 -0700
« prev ^ index » next coverage.py v7.4.3, created at 2024-03-14 11:41 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import unittest
25import lsst.geom
26from lsst.ip.diffim import detectAndMeasure, subtractImages
27from lsst.ip.diffim.utils import makeTestImage
28from lsst.pipe.base import InvalidQuantumError
29import lsst.utils.tests
32class DetectAndMeasureTestBase(lsst.utils.tests.TestCase):
34 def _check_diaSource(self, refSources, diaSource, refIds=None,
35 matchDistance=1., scale=1., usePsfFlux=True,
36 rtol=0.02, atol=None):
37 """Match a diaSource with a source in a reference catalog
38 and compare properties.
40 Parameters
41 ----------
42 refSources : `lsst.afw.table.SourceCatalog`
43 The reference catalog.
44 diaSource : `lsst.afw.table.SourceRecord`
45 The new diaSource to match to the reference catalog.
46 refIds : `list` of `int`, optional
47 Source IDs of previously associated diaSources.
48 matchDistance : `float`, optional
49 Maximum distance allowed between the detected and reference source
50 locations, in pixels.
51 scale : `float`, optional
52 Optional factor to scale the flux by before performing the test.
53 usePsfFlux : `bool`, optional
54 If set, test the PsfInstFlux field, otherwise use ApInstFlux.
55 rtol : `float`, optional
56 Relative tolerance of the flux value test.
57 atol : `float`, optional
58 Absolute tolerance of the flux value test.
59 """
60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2
61 + (diaSource.getY() - refSources.getY())**2)
62 self.assertLess(min(distance), matchDistance)
63 src = refSources[np.argmin(distance)]
64 if refIds is not None:
65 # Check that the same source was not previously associated
66 self.assertNotIn(src.getId(), refIds)
67 refIds.append(src.getId())
68 if atol is None:
69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux()
70 if usePsfFlux:
71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(),
72 rtol=rtol, atol=atol)
73 else:
74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(),
75 rtol=rtol, atol=atol)
77 def _check_values(self, values, minValue=None, maxValue=None):
78 """Verify that an array has finite values, and optionally that they are
79 within specified minimum and maximum bounds.
81 Parameters
82 ----------
83 values : `numpy.ndarray`
84 Array of values to check.
85 minValue : `float`, optional
86 Minimum allowable value.
87 maxValue : `float`, optional
88 Maximum allowable value.
89 """
90 self.assertTrue(np.all(np.isfinite(values)))
91 if minValue is not None:
92 self.assertTrue(np.all(values >= minValue))
93 if maxValue is not None:
94 self.assertTrue(np.all(values <= maxValue))
96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs):
97 """Setup and configure the detection and measurement PipelineTask.
99 Parameters
100 ----------
101 doSkySources : `bool`, optional
102 Generate sky sources.
103 nSkySources : `int`, optional
104 The number of sky sources to add in isolated background regions.
105 **kwargs
106 Any additional config parameters to set.
108 Returns
109 -------
110 `lsst.pipe.base.PipelineTask`
111 The configured Task to use for detection and measurement.
112 """
113 config = self.detectionTask.ConfigClass()
114 config.doSkySources = doSkySources
115 if doSkySources:
116 config.skySources.nSources = nSkySources
117 config.update(**kwargs)
118 return self.detectionTask(config=config)
121class DetectAndMeasureTest(DetectAndMeasureTestBase):
122 detectionTask = detectAndMeasure.DetectAndMeasureTask
124 def test_detection_xy0(self):
125 """Basic functionality test with non-zero x0 and y0.
126 """
127 # Set up the simulated images
128 noiseLevel = 1.
129 staticSeed = 1
130 fluxLevel = 500
131 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
132 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
133 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
134 difference = science.clone()
136 # Configure the detection Task
137 detectionTask = self._setup_detection()
139 # Run detection and check the results
140 output = detectionTask.run(science, matchedTemplate, difference)
141 subtractedMeasuredExposure = output.subtractedMeasuredExposure
143 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
145 def test_measurements_finite(self):
146 """Measured fluxes and centroids should always be finite.
147 """
148 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
150 # Set up the simulated images
151 noiseLevel = 1.
152 staticSeed = 1
153 transientSeed = 6
154 xSize = 256
155 ySize = 256
156 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
157 "xSize": xSize, "ySize": ySize}
158 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
159 nSrc=1, **kwargs)
160 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
161 nSrc=1, **kwargs)
162 rng = np.random.RandomState(3)
163 xLoc = np.arange(-5, xSize+5, 10)
164 rng.shuffle(xLoc)
165 yLoc = np.arange(-5, ySize+5, 10)
166 rng.shuffle(yLoc)
167 transients, transientSources = makeTestImage(seed=transientSeed,
168 nSrc=len(xLoc), fluxLevel=1000.,
169 noiseLevel=noiseLevel, noiseSeed=8,
170 xLoc=xLoc, yLoc=yLoc,
171 **kwargs)
172 difference = science.clone()
173 difference.maskedImage -= matchedTemplate.maskedImage
174 difference.maskedImage += transients.maskedImage
176 # Configure the detection Task
177 detectionTask = self._setup_detection(doForcedMeasurement=True)
179 # Run detection and check the results
180 output = detectionTask.run(science, matchedTemplate, difference)
182 for column in columnNames:
183 self._check_values(output.diaSources[column])
184 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
185 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
186 self._check_values(output.diaSources.getPsfInstFlux())
188 def test_raise_config_schema_mismatch(self):
189 """Check that sources with specified flags are removed from the catalog.
190 """
191 # Configure the detection Task, and and set a config that is not in the schema
192 with self.assertRaises(InvalidQuantumError):
193 self._setup_detection(badSourceFlags=["Bogus_flag_42"])
195 def test_remove_unphysical(self):
196 """Check that sources with specified flags are removed from the catalog.
197 """
198 # Set up the simulated images
199 noiseLevel = 1.
200 staticSeed = 1
201 xSize = 256
202 ySize = 256
203 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize}
204 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
205 nSrc=1, **kwargs)
206 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
207 nSrc=1, **kwargs)
208 difference = science.clone()
209 bbox = difference.getBBox()
210 difference.maskedImage -= matchedTemplate.maskedImage
212 # Configure the detection Task, and do not remove unphysical sources
213 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
214 badSourceFlags=[])
216 # Run detection and check the results
217 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
218 badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
219 nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove)
220 # Verify that unphysical sources exist
221 self.assertGreater(nBadNoRemove, 0)
223 # Configure the detection Task, and remove unphysical sources
224 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
225 badSourceFlags=["base_PixelFlags_flag_offimage", ])
227 # Run detection and check the results
228 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
229 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
230 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove)
231 # Verify that all sources are physical
232 self.assertEqual(nBadDoRemove, 0)
233 # Set a few centroids outside the image bounding box
234 nSetBad = 5
235 for src in diaSources[0: nSetBad]:
236 src["slot_Centroid_x"] += xSize
237 src["slot_Centroid_y"] += ySize
238 src["base_PixelFlags_flag_offimage"] = True
239 # Verify that these sources are outside the image
240 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY())
241 nBad = np.count_nonzero(badDiaSrc)
242 self.assertEqual(nBad, nSetBad)
243 diaSourcesNoBad = detectionTask._removeBadSources(diaSources)
244 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY())
246 # Verify that no sources outside the image bounding box remain
247 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0)
248 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad)
250 def test_detect_transients(self):
251 """Run detection on a difference image containing transients.
252 """
253 # Set up the simulated images
254 noiseLevel = 1.
255 staticSeed = 1
256 transientSeed = 6
257 fluxLevel = 500
258 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
259 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
260 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
262 # Configure the detection Task
263 detectionTask = self._setup_detection(doMerge=False)
264 kwargs["seed"] = transientSeed
265 kwargs["nSrc"] = 10
266 kwargs["fluxLevel"] = 1000
268 # Run detection and check the results
269 def _detection_wrapper(positive=True):
270 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
271 difference = science.clone()
272 difference.maskedImage -= matchedTemplate.maskedImage
273 if positive:
274 difference.maskedImage += transients.maskedImage
275 else:
276 difference.maskedImage -= transients.maskedImage
277 output = detectionTask.run(science, matchedTemplate, difference)
278 refIds = []
279 scale = 1. if positive else -1.
280 for diaSource in output.diaSources:
281 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
282 _detection_wrapper(positive=True)
283 _detection_wrapper(positive=False)
285 def test_missing_mask_planes(self):
286 """Check that detection runs with missing mask planes.
287 """
288 # Set up the simulated images
289 noiseLevel = 1.
290 fluxLevel = 500
291 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []}
292 # Use different seeds for the science and template so every source is a diaSource
293 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
294 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
296 difference = science.clone()
297 difference.maskedImage -= matchedTemplate.maskedImage
298 detectionTask = self._setup_detection()
300 # Verify that detection runs without errors
301 detectionTask.run(science, matchedTemplate, difference)
303 def test_detect_dipoles(self):
304 """Run detection on a difference image containing dipoles.
305 """
306 # Set up the simulated images
307 noiseLevel = 1.
308 staticSeed = 1
309 fluxLevel = 1000
310 fluxRange = 1.5
311 nSources = 10
312 offset = 1
313 xSize = 300
314 ySize = 300
315 kernelSize = 32
316 # Avoid placing sources near the edge for this test, so that we can
317 # easily check that the correct number of sources are detected.
318 templateBorderSize = kernelSize//2
319 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
320 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
321 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
322 "xSize": xSize, "ySize": ySize}
323 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
324 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
325 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
326 difference = science.clone()
327 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
328 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
329 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
330 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
332 # Configure the detection Task
333 detectionTask = self._setup_detection(doMerge=False)
335 # Run detection and check the results
336 output = detectionTask.run(science, matchedTemplate, difference)
337 self.assertIn(dipoleFlag, output.diaSources.schema.getNames())
338 nSourcesDet = len(sources)
339 self.assertEqual(len(output.diaSources), 2*nSourcesDet)
340 refIds = []
341 # The diaSource check should fail if we don't merge positive and negative footprints
342 for diaSource in output.diaSources:
343 with self.assertRaises(AssertionError):
344 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
345 atol=np.sqrt(fluxRange*fluxLevel))
347 detectionTask2 = self._setup_detection(doMerge=True)
348 output2 = detectionTask2.run(science, matchedTemplate, difference)
349 self.assertEqual(len(output2.diaSources), nSourcesDet)
350 refIds = []
351 for diaSource in output2.diaSources:
352 if diaSource[dipoleFlag]:
353 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
354 rtol=0.05, atol=None, usePsfFlux=False)
355 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
356 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
357 else:
358 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
360 def test_sky_sources(self):
361 """Add sky sources and check that they are sufficiently far from other
362 sources and have negligible flux.
363 """
364 # Set up the simulated images
365 noiseLevel = 1.
366 staticSeed = 1
367 transientSeed = 6
368 transientFluxLevel = 1000.
369 transientFluxRange = 1.5
370 fluxLevel = 500
371 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
372 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
373 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
374 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
375 nSrc=10, fluxLevel=transientFluxLevel,
376 fluxRange=transientFluxRange,
377 noiseLevel=noiseLevel, noiseSeed=8)
378 difference = science.clone()
379 difference.maskedImage -= matchedTemplate.maskedImage
380 difference.maskedImage += transients.maskedImage
381 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2
383 # Configure the detection Task
384 detectionTask = self._setup_detection(doSkySources=True)
386 # Run detection and check the results
387 output = detectionTask.run(science, matchedTemplate, difference)
388 skySources = output.diaSources[output.diaSources["sky_source"]]
389 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources)
390 for skySource in skySources:
391 # The sky sources should not be close to any other source
392 with self.assertRaises(AssertionError):
393 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
394 with self.assertRaises(AssertionError):
395 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
396 # The sky sources should have low flux levels.
397 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
398 atol=np.sqrt(transientFluxRange*transientFluxLevel))
400 def test_edge_detections(self):
401 """Sources with certain bad mask planes set should not be detected.
402 """
403 # Set up the simulated images
404 noiseLevel = 1.
405 staticSeed = 1
406 transientSeed = 6
407 fluxLevel = 500
408 radius = 2
409 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
410 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
411 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
413 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask
414 # Configure the detection Task
415 detectionTask = self._setup_detection()
416 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
417 nBad = len(excludeMaskPlanes)
418 self.assertGreater(nBad, 0)
419 kwargs["seed"] = transientSeed
420 kwargs["nSrc"] = nBad
421 kwargs["fluxLevel"] = 1000
423 # Run detection and check the results
424 def _detection_wrapper(setFlags=True):
425 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
426 difference = science.clone()
427 difference.maskedImage -= matchedTemplate.maskedImage
428 difference.maskedImage += transients.maskedImage
429 if setFlags:
430 for src, badMask in zip(transientSources, excludeMaskPlanes):
431 srcX = int(src.getX())
432 srcY = int(src.getY())
433 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
434 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
435 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
436 output = detectionTask.run(science, matchedTemplate, difference)
437 refIds = []
438 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes)
439 if setFlags:
440 self.assertEqual(np.sum(~goodSrcFlags), nBad)
441 else:
442 self.assertEqual(np.sum(~goodSrcFlags), 0)
443 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
444 if ~goodSrcFlag:
445 with self.assertRaises(AssertionError):
446 self._check_diaSource(transientSources, diaSource, refIds=refIds)
447 else:
448 self._check_diaSource(transientSources, diaSource, refIds=refIds)
449 _detection_wrapper(setFlags=False)
450 _detection_wrapper(setFlags=True)
452 def test_fake_mask_plane_propagation(self):
453 """Test that we have the mask planes related to fakes in diffim images.
454 This is testing method called updateMasks
455 """
456 xSize = 256
457 ySize = 256
458 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
459 science_fake_img, science_fake_sources = makeTestImage(
460 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1
461 )
462 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
463 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
464 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1
465 )
466 # created fakes and added them to the images
467 science.image += science_fake_img.image
468 template.image += tmplt_fake_img.image
470 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
471 # adding mask planes to both science and template images
472 science.mask.addMaskPlane("FAKE")
473 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE")
474 template.mask.addMaskPlane("FAKE")
475 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE")
477 for a_science_source in science_fake_sources:
478 bbox = a_science_source.getFootprint().getBBox()
479 science[bbox].mask.array |= science_fake_bitmask
481 for a_template_source in tmplt_fake_sources:
482 bbox = a_template_source.getFootprint().getBBox()
483 template[bbox].mask.array |= template_fake_bitmask
485 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0
486 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0
488 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass()
489 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig)
490 subtraction = subtractTask.run(template, science, sources)
492 # check subtraction mask plane is set where we set the previous masks
493 diff_mask = subtraction.difference.mask
495 # science mask should be now in INJECTED
496 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
498 # template mask should be now in INJECTED_TEMPLATE
499 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
501 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int))
502 # The template is convolved, so the INJECTED_TEMPLATE mask plane may
503 # include more pixels than the FAKE mask plane
504 injTmplt_masked &= template_fake_masked
505 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int))
507 # Now check that detection of fakes have the correct flag for injections
508 detectionTask = self._setup_detection()
509 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
510 nBad = len(excludeMaskPlanes)
511 self.assertEqual(nBad, 1)
513 output = detectionTask.run(subtraction.matchedScience,
514 subtraction.matchedTemplate,
515 subtraction.difference)
517 sci_refIds = []
518 tmpl_refIds = []
519 for diaSrc in output.diaSources:
520 if diaSrc['base_PsfFlux_instFlux'] > 0:
521 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds)
522 self.assertTrue(diaSrc['base_PixelFlags_flag_injected'])
523 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter'])
524 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template'])
525 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
526 else:
527 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds)
528 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template'])
529 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
530 self.assertFalse(diaSrc['base_PixelFlags_flag_injected'])
531 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter'])
534class DetectAndMeasureScoreTest(DetectAndMeasureTestBase):
535 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask
537 def test_detection_xy0(self):
538 """Basic functionality test with non-zero x0 and y0.
539 """
540 # Set up the simulated images
541 noiseLevel = 1.
542 staticSeed = 1
543 fluxLevel = 500
544 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
545 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
546 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
547 difference = science.clone()
548 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
549 scienceKernel = science.psf.getKernel()
550 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
552 # Configure the detection Task
553 detectionTask = self._setup_detection()
555 # Run detection and check the results
556 output = detectionTask.run(science, matchedTemplate, difference, score)
557 subtractedMeasuredExposure = output.subtractedMeasuredExposure
559 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
561 def test_measurements_finite(self):
562 """Measured fluxes and centroids should always be finite.
563 """
564 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
566 # Set up the simulated images
567 noiseLevel = 1.
568 staticSeed = 1
569 transientSeed = 6
570 xSize = 256
571 ySize = 256
572 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
573 "xSize": xSize, "ySize": ySize}
574 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
575 nSrc=1, **kwargs)
576 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
577 nSrc=1, **kwargs)
578 rng = np.random.RandomState(3)
579 xLoc = np.arange(-5, xSize+5, 10)
580 rng.shuffle(xLoc)
581 yLoc = np.arange(-5, ySize+5, 10)
582 rng.shuffle(yLoc)
583 transients, transientSources = makeTestImage(seed=transientSeed,
584 nSrc=len(xLoc), fluxLevel=1000.,
585 noiseLevel=noiseLevel, noiseSeed=8,
586 xLoc=xLoc, yLoc=yLoc,
587 **kwargs)
588 difference = science.clone()
589 difference.maskedImage -= matchedTemplate.maskedImage
590 difference.maskedImage += transients.maskedImage
591 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
592 scienceKernel = science.psf.getKernel()
593 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
595 # Configure the detection Task
596 detectionTask = self._setup_detection(doForcedMeasurement=True)
598 # Run detection and check the results
599 output = detectionTask.run(science, matchedTemplate, difference, score)
601 for column in columnNames:
602 self._check_values(output.diaSources[column])
603 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
604 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
605 self._check_values(output.diaSources.getPsfInstFlux())
607 def test_detect_transients(self):
608 """Run detection on a difference image containing transients.
609 """
610 # Set up the simulated images
611 noiseLevel = 1.
612 staticSeed = 1
613 transientSeed = 6
614 fluxLevel = 500
615 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
616 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
617 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
618 scienceKernel = science.psf.getKernel()
619 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
621 # Configure the detection Task
622 detectionTask = self._setup_detection(doMerge=False)
623 kwargs["seed"] = transientSeed
624 kwargs["nSrc"] = 10
625 kwargs["fluxLevel"] = 1000
627 # Run detection and check the results
628 def _detection_wrapper(positive=True):
629 """Simulate positive or negative transients and run detection.
631 Parameters
632 ----------
633 positive : `bool`, optional
634 If set, use positive transient sources.
635 """
637 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
638 difference = science.clone()
639 difference.maskedImage -= matchedTemplate.maskedImage
640 if positive:
641 difference.maskedImage += transients.maskedImage
642 else:
643 difference.maskedImage -= transients.maskedImage
644 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
645 output = detectionTask.run(science, matchedTemplate, difference, score)
646 refIds = []
647 scale = 1. if positive else -1.
648 goodSrcFlags = subtractTask._checkMask(score.mask, transientSources,
649 subtractTask.config.badMaskPlanes)
650 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
651 if ~goodSrcFlag:
652 with self.assertRaises(AssertionError):
653 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
654 else:
655 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
656 _detection_wrapper(positive=True)
657 _detection_wrapper(positive=False)
659 def test_detect_dipoles(self):
660 """Run detection on a difference image containing dipoles.
661 """
662 # Set up the simulated images
663 noiseLevel = 1.
664 staticSeed = 1
665 fluxLevel = 1000
666 fluxRange = 1.5
667 nSources = 10
668 offset = 1
669 xSize = 300
670 ySize = 300
671 kernelSize = 32
672 # Avoid placing sources near the edge for this test, so that we can
673 # easily check that the correct number of sources are detected.
674 templateBorderSize = kernelSize//2
675 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
676 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
677 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
678 "xSize": xSize, "ySize": ySize}
679 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
680 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
681 difference = science.clone()
682 # Shift the template by a pixel in order to make dipoles in the difference image.
683 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
684 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
685 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
686 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
687 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
688 scienceKernel = science.psf.getKernel()
689 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
691 # Configure the detection Task
692 detectionTask = self._setup_detection(doMerge=False)
694 # Run detection and check the results
695 output = detectionTask.run(science, matchedTemplate, difference, score)
696 self.assertIn(dipoleFlag, output.diaSources.schema.getNames())
697 nSourcesDet = len(sources)
698 # Since we did not merge the dipoles, each source should result in
699 # both a positive and a negative diaSource
700 self.assertEqual(len(output.diaSources), 2*nSourcesDet)
701 refIds = []
702 # The diaSource check should fail if we don't merge positive and negative footprints
703 for diaSource in output.diaSources:
704 with self.assertRaises(AssertionError):
705 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
706 atol=np.sqrt(fluxRange*fluxLevel))
708 detectionTask2 = self._setup_detection(doMerge=True)
709 output2 = detectionTask2.run(science, matchedTemplate, difference, score)
710 self.assertEqual(len(output2.diaSources), nSourcesDet)
711 refIds = []
712 for diaSource in output2.diaSources:
713 if diaSource[dipoleFlag]:
714 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
715 rtol=0.05, atol=None, usePsfFlux=False)
716 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
717 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
718 else:
719 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
721 def test_sky_sources(self):
722 """Add sky sources and check that they are sufficiently far from other
723 sources and have negligible flux.
724 """
725 # Set up the simulated images
726 noiseLevel = 1.
727 staticSeed = 1
728 transientSeed = 6
729 transientFluxLevel = 1000.
730 transientFluxRange = 1.5
731 fluxLevel = 500
732 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
733 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
734 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
735 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
736 nSrc=10, fluxLevel=transientFluxLevel,
737 fluxRange=transientFluxRange,
738 noiseLevel=noiseLevel, noiseSeed=8)
739 difference = science.clone()
740 difference.maskedImage -= matchedTemplate.maskedImage
741 difference.maskedImage += transients.maskedImage
742 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
743 scienceKernel = science.psf.getKernel()
744 kernelWidth = np.max(scienceKernel.getDimensions())//2
745 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
747 # Configure the detection Task
748 detectionTask = self._setup_detection(doSkySources=True)
750 # Run detection and check the results
751 output = detectionTask.run(science, matchedTemplate, difference, score)
752 nSkySourcesGenerated = detectionTask.metadata["nSkySources"]
753 skySources = output.diaSources[output.diaSources["sky_source"]]
754 self.assertEqual(len(skySources), nSkySourcesGenerated)
755 for skySource in skySources:
756 # The sky sources should not be close to any other source
757 with self.assertRaises(AssertionError):
758 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
759 with self.assertRaises(AssertionError):
760 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
761 # The sky sources should have low flux levels.
762 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
763 atol=np.sqrt(transientFluxRange*transientFluxLevel))
765 def test_edge_detections(self):
766 """Sources with certain bad mask planes set should not be detected.
767 """
768 # Set up the simulated images
769 noiseLevel = 1.
770 staticSeed = 1
771 transientSeed = 6
772 fluxLevel = 500
773 radius = 2
774 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
775 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
776 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
778 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
779 scienceKernel = science.psf.getKernel()
780 # Configure the detection Task
781 detectionTask = self._setup_detection()
782 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
783 nBad = len(excludeMaskPlanes)
784 self.assertGreater(nBad, 0)
785 kwargs["seed"] = transientSeed
786 kwargs["nSrc"] = nBad
787 kwargs["fluxLevel"] = 1000
789 # Run detection and check the results
790 def _detection_wrapper(setFlags=True):
791 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
792 difference = science.clone()
793 difference.maskedImage -= matchedTemplate.maskedImage
794 difference.maskedImage += transients.maskedImage
795 if setFlags:
796 for src, badMask in zip(transientSources, excludeMaskPlanes):
797 srcX = int(src.getX())
798 srcY = int(src.getY())
799 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
800 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
801 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
802 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
803 output = detectionTask.run(science, matchedTemplate, difference, score)
804 refIds = []
805 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes)
806 if setFlags:
807 self.assertEqual(np.sum(~goodSrcFlags), nBad)
808 else:
809 self.assertEqual(np.sum(~goodSrcFlags), 0)
810 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
811 if ~goodSrcFlag:
812 with self.assertRaises(AssertionError):
813 self._check_diaSource(transientSources, diaSource, refIds=refIds)
814 else:
815 self._check_diaSource(transientSources, diaSource, refIds=refIds)
816 _detection_wrapper(setFlags=False)
817 _detection_wrapper(setFlags=True)
820def setup_module(module):
821 lsst.utils.tests.init()
824class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
825 pass
828if __name__ == "__main__": 828 ↛ 829line 828 didn't jump to line 829, because the condition on line 828 was never true
829 lsst.utils.tests.init()
830 unittest.main()