Coverage for tests/test_detectAndMeasure.py: 6%
507 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-07 03:33 -0700
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-07 03:33 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import unittest
25import lsst.geom
26from lsst.ip.diffim import detectAndMeasure, subtractImages
27from lsst.ip.diffim.utils import makeTestImage
28from lsst.pipe.base import InvalidQuantumError
29import lsst.utils.tests
32class DetectAndMeasureTestBase:
34 def _check_diaSource(self, refSources, diaSource, refIds=None,
35 matchDistance=1., scale=1., usePsfFlux=True,
36 rtol=0.025, atol=None):
37 """Match a diaSource with a source in a reference catalog
38 and compare properties.
40 Parameters
41 ----------
42 refSources : `lsst.afw.table.SourceCatalog`
43 The reference catalog.
44 diaSource : `lsst.afw.table.SourceRecord`
45 The new diaSource to match to the reference catalog.
46 refIds : `list` of `int`, optional
47 Source IDs of previously associated diaSources.
48 matchDistance : `float`, optional
49 Maximum distance allowed between the detected and reference source
50 locations, in pixels.
51 scale : `float`, optional
52 Optional factor to scale the flux by before performing the test.
53 usePsfFlux : `bool`, optional
54 If set, test the PsfInstFlux field, otherwise use ApInstFlux.
55 rtol : `float`, optional
56 Relative tolerance of the flux value test.
57 atol : `float`, optional
58 Absolute tolerance of the flux value test.
59 """
60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2
61 + (diaSource.getY() - refSources.getY())**2)
62 self.assertLess(min(distance), matchDistance)
63 src = refSources[np.argmin(distance)]
64 if refIds is not None:
65 # Check that the same source was not previously associated
66 self.assertNotIn(src.getId(), refIds)
67 refIds.append(src.getId())
68 if atol is None:
69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux()
70 if usePsfFlux:
71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(),
72 rtol=rtol, atol=atol)
73 else:
74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(),
75 rtol=rtol, atol=atol)
77 def _check_values(self, values, minValue=None, maxValue=None):
78 """Verify that an array has finite values, and optionally that they are
79 within specified minimum and maximum bounds.
81 Parameters
82 ----------
83 values : `numpy.ndarray`
84 Array of values to check.
85 minValue : `float`, optional
86 Minimum allowable value.
87 maxValue : `float`, optional
88 Maximum allowable value.
89 """
90 self.assertTrue(np.all(np.isfinite(values)))
91 if minValue is not None:
92 self.assertTrue(np.all(values >= minValue))
93 if maxValue is not None:
94 self.assertTrue(np.all(values <= maxValue))
96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs):
97 """Setup and configure the detection and measurement PipelineTask.
99 Parameters
100 ----------
101 doSkySources : `bool`, optional
102 Generate sky sources.
103 nSkySources : `int`, optional
104 The number of sky sources to add in isolated background regions.
105 **kwargs
106 Any additional config parameters to set.
108 Returns
109 -------
110 `lsst.pipe.base.PipelineTask`
111 The configured Task to use for detection and measurement.
112 """
113 config = self.detectionTask.ConfigClass()
114 config.doSkySources = doSkySources
115 if doSkySources:
116 config.skySources.nSources = nSkySources
117 config.update(**kwargs)
119 # Make a realistic id generator so that output catalog ids are useful.
120 dataId = lsst.daf.butler.DataCoordinate.standardize(
121 instrument="I",
122 visit=42,
123 detector=12,
124 universe=lsst.daf.butler.DimensionUniverse(),
125 )
126 config.idGenerator.packer.name = "observation"
127 config.idGenerator.packer["observation"].n_observations = 10000
128 config.idGenerator.packer["observation"].n_detectors = 99
129 config.idGenerator.n_releases = 8
130 config.idGenerator.release_id = 2
131 self.idGenerator = config.idGenerator.apply(dataId)
133 return self.detectionTask(config=config)
136class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
137 detectionTask = detectAndMeasure.DetectAndMeasureTask
139 def test_detection_xy0(self):
140 """Basic functionality test with non-zero x0 and y0.
141 """
142 # Set up the simulated images
143 noiseLevel = 1.
144 staticSeed = 1
145 fluxLevel = 500
146 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
147 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
148 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
149 difference = science.clone()
151 # Configure the detection Task
152 detectionTask = self._setup_detection()
154 # Run detection and check the results
155 output = detectionTask.run(science, matchedTemplate, difference,
156 idFactory=self.idGenerator.make_table_id_factory())
157 subtractedMeasuredExposure = output.subtractedMeasuredExposure
159 # Catalog ids should be very large from this id generator.
160 self.assertTrue(all(output.diaSources['id'] > 1000000000))
161 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
163 def test_measurements_finite(self):
164 """Measured fluxes and centroids should always be finite.
165 """
166 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
168 # Set up the simulated images
169 noiseLevel = 1.
170 staticSeed = 1
171 transientSeed = 6
172 xSize = 256
173 ySize = 256
174 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
175 "xSize": xSize, "ySize": ySize}
176 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
177 nSrc=1, **kwargs)
178 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
179 nSrc=1, **kwargs)
180 rng = np.random.RandomState(3)
181 xLoc = np.arange(-5, xSize+5, 10)
182 rng.shuffle(xLoc)
183 yLoc = np.arange(-5, ySize+5, 10)
184 rng.shuffle(yLoc)
185 transients, transientSources = makeTestImage(seed=transientSeed,
186 nSrc=len(xLoc), fluxLevel=1000.,
187 noiseLevel=noiseLevel, noiseSeed=8,
188 xLoc=xLoc, yLoc=yLoc,
189 **kwargs)
190 difference = science.clone()
191 difference.maskedImage -= matchedTemplate.maskedImage
192 difference.maskedImage += transients.maskedImage
194 # Configure the detection Task
195 detectionTask = self._setup_detection(doForcedMeasurement=True)
197 # Run detection and check the results
198 output = detectionTask.run(science, matchedTemplate, difference)
200 for column in columnNames:
201 self._check_values(output.diaSources[column])
202 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
203 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
204 self._check_values(output.diaSources.getPsfInstFlux())
206 def test_raise_config_schema_mismatch(self):
207 """Check that sources with specified flags are removed from the catalog.
208 """
209 # Configure the detection Task, and and set a config that is not in the schema
210 with self.assertRaises(InvalidQuantumError):
211 self._setup_detection(badSourceFlags=["Bogus_flag_42"])
213 def test_remove_unphysical(self):
214 """Check that sources with specified flags are removed from the catalog.
215 """
216 # Set up the simulated images
217 noiseLevel = 1.
218 staticSeed = 1
219 xSize = 256
220 ySize = 256
221 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize}
222 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
223 nSrc=1, **kwargs)
224 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
225 nSrc=1, **kwargs)
226 difference = science.clone()
227 bbox = difference.getBBox()
228 difference.maskedImage -= matchedTemplate.maskedImage
230 # Configure the detection Task, and remove unphysical sources
231 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
232 badSourceFlags=["base_PixelFlags_flag_offimage", ])
234 # Run detection and check the results
235 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
236 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
237 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove)
238 # Verify that all sources are physical
239 self.assertEqual(nBadDoRemove, 0)
240 # Set a few centroids outside the image bounding box
241 nSetBad = 5
242 for src in diaSources[0: nSetBad]:
243 src["slot_Centroid_x"] += xSize
244 src["slot_Centroid_y"] += ySize
245 src["base_PixelFlags_flag_offimage"] = True
246 # Verify that these sources are outside the image
247 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY())
248 nBad = np.count_nonzero(badDiaSrc)
249 self.assertEqual(nBad, nSetBad)
250 diaSourcesNoBad = detectionTask._removeBadSources(diaSources)
251 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY())
253 # Verify that no sources outside the image bounding box remain
254 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0)
255 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad)
257 def test_detect_transients(self):
258 """Run detection on a difference image containing transients.
259 """
260 # Set up the simulated images
261 noiseLevel = 1.
262 staticSeed = 1
263 transientSeed = 6
264 fluxLevel = 500
265 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
266 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
267 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
269 # Configure the detection Task
270 detectionTask = self._setup_detection(doMerge=False)
271 kwargs["seed"] = transientSeed
272 kwargs["nSrc"] = 10
273 kwargs["fluxLevel"] = 1000
275 # Run detection and check the results
276 def _detection_wrapper(positive=True):
277 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
278 difference = science.clone()
279 difference.maskedImage -= matchedTemplate.maskedImage
280 if positive:
281 difference.maskedImage += transients.maskedImage
282 else:
283 difference.maskedImage -= transients.maskedImage
284 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
285 # science image if we've e.g. removed parents post-deblending.
286 # Pass a clone of the science image, so that it doesn't disrupt
287 # later tests.
288 output = detectionTask.run(science.clone(), matchedTemplate, difference)
289 refIds = []
290 scale = 1. if positive else -1.
291 for diaSource in output.diaSources:
292 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
293 _detection_wrapper(positive=True)
294 _detection_wrapper(positive=False)
296 def test_missing_mask_planes(self):
297 """Check that detection runs with missing mask planes.
298 """
299 # Set up the simulated images
300 noiseLevel = 1.
301 fluxLevel = 500
302 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []}
303 # Use different seeds for the science and template so every source is a diaSource
304 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
305 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
307 difference = science.clone()
308 difference.maskedImage -= matchedTemplate.maskedImage
309 detectionTask = self._setup_detection()
311 # Verify that detection runs without errors
312 detectionTask.run(science, matchedTemplate, difference)
314 def test_detect_dipoles(self):
315 """Run detection on a difference image containing dipoles.
316 """
317 # Set up the simulated images
318 noiseLevel = 1.
319 staticSeed = 1
320 fluxLevel = 1000
321 fluxRange = 1.5
322 nSources = 10
323 offset = 1
324 xSize = 300
325 ySize = 300
326 kernelSize = 32
327 # Avoid placing sources near the edge for this test, so that we can
328 # easily check that the correct number of sources are detected.
329 templateBorderSize = kernelSize//2
330 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
331 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
332 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
333 "xSize": xSize, "ySize": ySize}
334 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
335 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
336 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
337 difference = science.clone()
338 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
339 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
340 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
341 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
343 detectionTask = self._setup_detection(doMerge=True)
344 output = detectionTask.run(science, matchedTemplate, difference)
345 self.assertEqual(len(output.diaSources), len(sources))
346 refIds = []
347 for diaSource in output.diaSources:
348 if diaSource[dipoleFlag]:
349 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
350 rtol=0.05, atol=None, usePsfFlux=False)
351 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
352 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
353 else:
354 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
356 def test_sky_sources(self):
357 """Add sky sources and check that they are sufficiently far from other
358 sources and have negligible flux.
359 """
360 # Set up the simulated images
361 noiseLevel = 1.
362 staticSeed = 1
363 transientSeed = 6
364 transientFluxLevel = 1000.
365 transientFluxRange = 1.5
366 fluxLevel = 500
367 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
368 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
369 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
370 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
371 nSrc=10, fluxLevel=transientFluxLevel,
372 fluxRange=transientFluxRange,
373 noiseLevel=noiseLevel, noiseSeed=8)
374 difference = science.clone()
375 difference.maskedImage -= matchedTemplate.maskedImage
376 difference.maskedImage += transients.maskedImage
377 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2
379 # Configure the detection Task
380 detectionTask = self._setup_detection(doSkySources=True)
382 # Run detection and check the results
383 output = detectionTask.run(science, matchedTemplate, difference,
384 idFactory=self.idGenerator.make_table_id_factory())
385 skySources = output.diaSources[output.diaSources["sky_source"]]
386 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources)
387 for skySource in skySources:
388 # The sky sources should not be close to any other source
389 with self.assertRaises(AssertionError):
390 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
391 with self.assertRaises(AssertionError):
392 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
393 # The sky sources should have low flux levels.
394 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
395 atol=np.sqrt(transientFluxRange*transientFluxLevel))
397 # Catalog ids should be very large from this id generator.
398 self.assertTrue(all(output.diaSources['id'] > 1000000000))
400 def test_edge_detections(self):
401 """Sources with certain bad mask planes set should not be detected.
402 """
403 # Set up the simulated images
404 noiseLevel = 1.
405 staticSeed = 1
406 transientSeed = 6
407 fluxLevel = 500
408 radius = 2
409 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
410 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
411 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
413 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask
414 # Configure the detection Task
415 detectionTask = self._setup_detection()
416 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
417 nBad = len(excludeMaskPlanes)
418 self.assertGreater(nBad, 0)
419 kwargs["seed"] = transientSeed
420 kwargs["nSrc"] = nBad
421 kwargs["fluxLevel"] = 1000
423 # Run detection and check the results
424 def _detection_wrapper(setFlags=True):
425 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
426 difference = science.clone()
427 difference.maskedImage -= matchedTemplate.maskedImage
428 difference.maskedImage += transients.maskedImage
429 if setFlags:
430 for src, badMask in zip(transientSources, excludeMaskPlanes):
431 srcX = int(src.getX())
432 srcY = int(src.getY())
433 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
434 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
435 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
436 output = detectionTask.run(science, matchedTemplate, difference)
437 refIds = []
438 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes)
439 if setFlags:
440 self.assertEqual(np.sum(~goodSrcFlags), nBad)
441 else:
442 self.assertEqual(np.sum(~goodSrcFlags), 0)
443 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
444 if ~goodSrcFlag:
445 with self.assertRaises(AssertionError):
446 self._check_diaSource(transientSources, diaSource, refIds=refIds)
447 else:
448 self._check_diaSource(transientSources, diaSource, refIds=refIds)
449 _detection_wrapper(setFlags=False)
450 _detection_wrapper(setFlags=True)
452 def test_fake_mask_plane_propagation(self):
453 """Test that we have the mask planes related to fakes in diffim images.
454 This is testing method called updateMasks
455 """
456 xSize = 256
457 ySize = 256
458 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
459 science_fake_img, science_fake_sources = makeTestImage(
460 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1
461 )
462 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
463 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
464 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1
465 )
466 # created fakes and added them to the images
467 science.image += science_fake_img.image
468 template.image += tmplt_fake_img.image
470 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
471 # adding mask planes to both science and template images
472 science.mask.addMaskPlane("FAKE")
473 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE")
474 template.mask.addMaskPlane("FAKE")
475 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE")
477 # makeTestImage sets the DETECTED plane on the sources; we can use
478 # that to set the FAKE plane on the science and template images.
479 detected = science_fake_img.mask.getPlaneBitMask("DETECTED")
480 fake_pixels = (science_fake_img.mask.array & detected).nonzero()
481 science.mask.array[fake_pixels] |= science_fake_bitmask
482 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED")
483 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero()
484 template.mask.array[fake_pixels] |= science_fake_bitmask
486 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0
487 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0
489 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass()
490 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig)
491 subtraction = subtractTask.run(template, science, sources)
493 # check subtraction mask plane is set where we set the previous masks
494 diff_mask = subtraction.difference.mask
496 # science mask should be now in INJECTED
497 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
499 # template mask should be now in INJECTED_TEMPLATE
500 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
502 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int))
503 # The template is convolved, so the INJECTED_TEMPLATE mask plane may
504 # include more pixels than the FAKE mask plane
505 injTmplt_masked &= template_fake_masked
506 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int))
508 # Now check that detection of fakes have the correct flag for injections
509 detectionTask = self._setup_detection()
510 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
511 nBad = len(excludeMaskPlanes)
512 self.assertEqual(nBad, 1)
514 output = detectionTask.run(subtraction.matchedScience,
515 subtraction.matchedTemplate,
516 subtraction.difference)
518 sci_refIds = []
519 tmpl_refIds = []
520 for diaSrc in output.diaSources:
521 if diaSrc['base_PsfFlux_instFlux'] > 0:
522 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds)
523 self.assertTrue(diaSrc['base_PixelFlags_flag_injected'])
524 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter'])
525 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template'])
526 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
527 else:
528 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds)
529 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template'])
530 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
531 self.assertFalse(diaSrc['base_PixelFlags_flag_injected'])
532 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter'])
534 def test_mask_streaks(self):
535 """Run detection on a difference image containing a streak.
536 """
537 # Set up the simulated images
538 noiseLevel = 1.
539 staticSeed = 1
540 fluxLevel = 500
541 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
542 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
543 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
545 # Configure the detection Task
546 detectionTask = self._setup_detection(doMerge=False, doMaskStreaks=True)
548 # Test that no streaks are detected
549 difference = science.clone()
550 difference.maskedImage -= matchedTemplate.maskedImage
551 output = detectionTask.run(science, matchedTemplate, difference)
552 outMask = output.subtractedMeasuredExposure.mask.array
553 streakMask = output.subtractedMeasuredExposure.mask.getPlaneBitMask("STREAK")
554 streakMaskSet = (outMask & streakMask) > 0
555 self.assertTrue(np.all(streakMaskSet == 0))
557 # Add streak-like shape and check that streak is detected
558 difference.image.array[20:23, 40:200] += 50
559 output = detectionTask.run(science, matchedTemplate, difference)
560 outMask = output.subtractedMeasuredExposure.mask.array
561 streakMask = output.subtractedMeasuredExposure.mask.getPlaneBitMask("STREAK")
562 streakMaskSet = (outMask & streakMask) > 0
563 self.assertTrue(np.all(streakMaskSet[20:23, 40:200]))
566class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
567 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask
569 def test_detection_xy0(self):
570 """Basic functionality test with non-zero x0 and y0.
571 """
572 # Set up the simulated images
573 noiseLevel = 1.
574 staticSeed = 1
575 fluxLevel = 500
576 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
577 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
578 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
579 difference = science.clone()
580 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
581 scienceKernel = science.psf.getKernel()
582 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
584 # Configure the detection Task
585 detectionTask = self._setup_detection()
587 # Run detection and check the results
588 output = detectionTask.run(science, matchedTemplate, difference, score,
589 idFactory=self.idGenerator.make_table_id_factory())
591 # Catalog ids should be very large from this id generator.
592 self.assertTrue(all(output.diaSources['id'] > 1000000000))
593 subtractedMeasuredExposure = output.subtractedMeasuredExposure
595 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
597 def test_measurements_finite(self):
598 """Measured fluxes and centroids should always be finite.
599 """
600 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
602 # Set up the simulated images
603 noiseLevel = 1.
604 staticSeed = 1
605 transientSeed = 6
606 xSize = 256
607 ySize = 256
608 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
609 "xSize": xSize, "ySize": ySize}
610 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
611 nSrc=1, **kwargs)
612 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
613 nSrc=1, **kwargs)
614 rng = np.random.RandomState(3)
615 xLoc = np.arange(-5, xSize+5, 10)
616 rng.shuffle(xLoc)
617 yLoc = np.arange(-5, ySize+5, 10)
618 rng.shuffle(yLoc)
619 transients, transientSources = makeTestImage(seed=transientSeed,
620 nSrc=len(xLoc), fluxLevel=1000.,
621 noiseLevel=noiseLevel, noiseSeed=8,
622 xLoc=xLoc, yLoc=yLoc,
623 **kwargs)
624 difference = science.clone()
625 difference.maskedImage -= matchedTemplate.maskedImage
626 difference.maskedImage += transients.maskedImage
627 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
628 scienceKernel = science.psf.getKernel()
629 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
631 # Configure the detection Task
632 detectionTask = self._setup_detection(doForcedMeasurement=True)
634 # Run detection and check the results
635 output = detectionTask.run(science, matchedTemplate, difference, score)
637 for column in columnNames:
638 self._check_values(output.diaSources[column])
639 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
640 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
641 self._check_values(output.diaSources.getPsfInstFlux())
643 def test_detect_transients(self):
644 """Run detection on a difference image containing transients.
645 """
646 # Set up the simulated images
647 noiseLevel = 1.
648 staticSeed = 1
649 transientSeed = 6
650 fluxLevel = 500
651 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
652 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
653 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
654 scienceKernel = science.psf.getKernel()
655 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
657 # Configure the detection Task
658 detectionTask = self._setup_detection(doMerge=False)
659 kwargs["seed"] = transientSeed
660 kwargs["nSrc"] = 10
661 kwargs["fluxLevel"] = 1000
663 # Run detection and check the results
664 def _detection_wrapper(positive=True):
665 """Simulate positive or negative transients and run detection.
667 Parameters
668 ----------
669 positive : `bool`, optional
670 If set, use positive transient sources.
671 """
673 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
674 difference = science.clone()
675 difference.maskedImage -= matchedTemplate.maskedImage
676 if positive:
677 difference.maskedImage += transients.maskedImage
678 else:
679 difference.maskedImage -= transients.maskedImage
680 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
681 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
682 # science image if we've e.g. removed parents post-deblending.
683 # Pass a clone of the science image, so that it doesn't disrupt
684 # later tests.
685 output = detectionTask.run(science.clone(), matchedTemplate, difference, score)
686 refIds = []
687 scale = 1. if positive else -1.
688 # sources near the edge may have untrustworthy centroids
689 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge']
690 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
691 if goodSrcFlag:
692 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
693 _detection_wrapper(positive=True)
694 _detection_wrapper(positive=False)
696 def test_detect_dipoles(self):
697 """Run detection on a difference image containing dipoles.
698 """
699 # Set up the simulated images
700 noiseLevel = 1.
701 staticSeed = 1
702 fluxLevel = 1000
703 fluxRange = 1.5
704 nSources = 10
705 offset = 1
706 xSize = 300
707 ySize = 300
708 kernelSize = 32
709 # Avoid placing sources near the edge for this test, so that we can
710 # easily check that the correct number of sources are detected.
711 templateBorderSize = kernelSize//2
712 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
713 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
714 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
715 "xSize": xSize, "ySize": ySize}
716 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
717 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
718 difference = science.clone()
719 # Shift the template by a pixel in order to make dipoles in the difference image.
720 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
721 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
722 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
723 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
724 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
725 scienceKernel = science.psf.getKernel()
726 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
728 detectionTask = self._setup_detection()
729 output = detectionTask.run(science, matchedTemplate, difference, score)
730 self.assertEqual(len(output.diaSources), len(sources))
731 refIds = []
732 for diaSource in output.diaSources:
733 if diaSource[dipoleFlag]:
734 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
735 rtol=0.05, atol=None, usePsfFlux=False)
736 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
737 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
738 else:
739 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
741 def test_sky_sources(self):
742 """Add sky sources and check that they are sufficiently far from other
743 sources and have negligible flux.
744 """
745 # Set up the simulated images
746 noiseLevel = 1.
747 staticSeed = 1
748 transientSeed = 6
749 transientFluxLevel = 1000.
750 transientFluxRange = 1.5
751 fluxLevel = 500
752 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
753 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
754 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
755 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
756 nSrc=10, fluxLevel=transientFluxLevel,
757 fluxRange=transientFluxRange,
758 noiseLevel=noiseLevel, noiseSeed=8)
759 difference = science.clone()
760 difference.maskedImage -= matchedTemplate.maskedImage
761 difference.maskedImage += transients.maskedImage
762 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
763 scienceKernel = science.psf.getKernel()
764 kernelWidth = np.max(scienceKernel.getDimensions())//2
765 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
767 # Configure the detection Task
768 detectionTask = self._setup_detection(doSkySources=True)
770 # Run detection and check the results
771 output = detectionTask.run(science, matchedTemplate, difference, score,
772 idFactory=self.idGenerator.make_table_id_factory())
773 nSkySourcesGenerated = detectionTask.metadata["n_skySources"]
774 skySources = output.diaSources[output.diaSources["sky_source"]]
775 self.assertEqual(len(skySources), nSkySourcesGenerated)
776 for skySource in skySources:
777 # The sky sources should not be close to any other source
778 with self.assertRaises(AssertionError):
779 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
780 with self.assertRaises(AssertionError):
781 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
782 # The sky sources should have low flux levels.
783 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
784 atol=np.sqrt(transientFluxRange*transientFluxLevel))
786 # Catalog ids should be very large from this id generator.
787 self.assertTrue(all(output.diaSources['id'] > 1000000000))
789 def test_edge_detections(self):
790 """Sources with certain bad mask planes set should not be detected.
791 """
792 # Set up the simulated images
793 noiseLevel = 1.
794 staticSeed = 1
795 transientSeed = 6
796 fluxLevel = 500
797 radius = 2
798 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
799 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
800 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
802 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
803 scienceKernel = science.psf.getKernel()
804 # Configure the detection Task
805 detectionTask = self._setup_detection()
806 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
807 nBad = len(excludeMaskPlanes)
808 self.assertGreater(nBad, 0)
809 kwargs["seed"] = transientSeed
810 kwargs["nSrc"] = nBad
811 kwargs["fluxLevel"] = 1000
813 # Run detection and check the results
814 def _detection_wrapper(setFlags=True):
815 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
816 difference = science.clone()
817 difference.maskedImage -= matchedTemplate.maskedImage
818 difference.maskedImage += transients.maskedImage
819 if setFlags:
820 for src, badMask in zip(transientSources, excludeMaskPlanes):
821 srcX = int(src.getX())
822 srcY = int(src.getY())
823 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
824 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
825 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
826 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
827 output = detectionTask.run(science, matchedTemplate, difference, score)
828 refIds = []
829 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes)
830 if setFlags:
831 self.assertEqual(np.sum(~goodSrcFlags), nBad)
832 else:
833 self.assertEqual(np.sum(~goodSrcFlags), 0)
834 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
835 if ~goodSrcFlag:
836 with self.assertRaises(AssertionError):
837 self._check_diaSource(transientSources, diaSource, refIds=refIds)
838 else:
839 self._check_diaSource(transientSources, diaSource, refIds=refIds)
840 _detection_wrapper(setFlags=False)
841 _detection_wrapper(setFlags=True)
844def setup_module(module):
845 lsst.utils.tests.init()
848class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
849 pass
852if __name__ == "__main__": 852 ↛ 853line 852 didn't jump to line 853, because the condition on line 852 was never true
853 lsst.utils.tests.init()
854 unittest.main()