Coverage for tests/test_detectAndMeasure.py: 6%
486 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-02 02:14 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-02 02:14 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import unittest
25import lsst.geom
26from lsst.ip.diffim import detectAndMeasure, subtractImages
27from lsst.ip.diffim.utils import makeTestImage
28from lsst.pipe.base import InvalidQuantumError
29import lsst.utils.tests
32class DetectAndMeasureTestBase:
34 def _check_diaSource(self, refSources, diaSource, refIds=None,
35 matchDistance=1., scale=1., usePsfFlux=True,
36 rtol=0.025, atol=None):
37 """Match a diaSource with a source in a reference catalog
38 and compare properties.
40 Parameters
41 ----------
42 refSources : `lsst.afw.table.SourceCatalog`
43 The reference catalog.
44 diaSource : `lsst.afw.table.SourceRecord`
45 The new diaSource to match to the reference catalog.
46 refIds : `list` of `int`, optional
47 Source IDs of previously associated diaSources.
48 matchDistance : `float`, optional
49 Maximum distance allowed between the detected and reference source
50 locations, in pixels.
51 scale : `float`, optional
52 Optional factor to scale the flux by before performing the test.
53 usePsfFlux : `bool`, optional
54 If set, test the PsfInstFlux field, otherwise use ApInstFlux.
55 rtol : `float`, optional
56 Relative tolerance of the flux value test.
57 atol : `float`, optional
58 Absolute tolerance of the flux value test.
59 """
60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2
61 + (diaSource.getY() - refSources.getY())**2)
62 self.assertLess(min(distance), matchDistance)
63 src = refSources[np.argmin(distance)]
64 if refIds is not None:
65 # Check that the same source was not previously associated
66 self.assertNotIn(src.getId(), refIds)
67 refIds.append(src.getId())
68 if atol is None:
69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux()
70 if usePsfFlux:
71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(),
72 rtol=rtol, atol=atol)
73 else:
74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(),
75 rtol=rtol, atol=atol)
77 def _check_values(self, values, minValue=None, maxValue=None):
78 """Verify that an array has finite values, and optionally that they are
79 within specified minimum and maximum bounds.
81 Parameters
82 ----------
83 values : `numpy.ndarray`
84 Array of values to check.
85 minValue : `float`, optional
86 Minimum allowable value.
87 maxValue : `float`, optional
88 Maximum allowable value.
89 """
90 self.assertTrue(np.all(np.isfinite(values)))
91 if minValue is not None:
92 self.assertTrue(np.all(values >= minValue))
93 if maxValue is not None:
94 self.assertTrue(np.all(values <= maxValue))
96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs):
97 """Setup and configure the detection and measurement PipelineTask.
99 Parameters
100 ----------
101 doSkySources : `bool`, optional
102 Generate sky sources.
103 nSkySources : `int`, optional
104 The number of sky sources to add in isolated background regions.
105 **kwargs
106 Any additional config parameters to set.
108 Returns
109 -------
110 `lsst.pipe.base.PipelineTask`
111 The configured Task to use for detection and measurement.
112 """
113 config = self.detectionTask.ConfigClass()
114 config.doSkySources = doSkySources
115 if doSkySources:
116 config.skySources.nSources = nSkySources
117 config.update(**kwargs)
119 # Make a realistic id generator so that output catalog ids are useful.
120 dataId = lsst.daf.butler.DataCoordinate.standardize(
121 instrument="I",
122 visit=42,
123 detector=12,
124 universe=lsst.daf.butler.DimensionUniverse(),
125 )
126 config.idGenerator.packer.name = "observation"
127 config.idGenerator.packer["observation"].n_observations = 10000
128 config.idGenerator.packer["observation"].n_detectors = 99
129 config.idGenerator.n_releases = 8
130 config.idGenerator.release_id = 2
131 self.idGenerator = config.idGenerator.apply(dataId)
133 return self.detectionTask(config=config)
136class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
137 detectionTask = detectAndMeasure.DetectAndMeasureTask
139 def test_detection_xy0(self):
140 """Basic functionality test with non-zero x0 and y0.
141 """
142 # Set up the simulated images
143 noiseLevel = 1.
144 staticSeed = 1
145 fluxLevel = 500
146 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
147 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
148 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
149 difference = science.clone()
151 # Configure the detection Task
152 detectionTask = self._setup_detection()
154 # Run detection and check the results
155 output = detectionTask.run(science, matchedTemplate, difference,
156 idFactory=self.idGenerator.make_table_id_factory())
157 subtractedMeasuredExposure = output.subtractedMeasuredExposure
159 # Catalog ids should be very large from this id generator.
160 self.assertTrue(all(output.diaSources['id'] > 1000000000))
161 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
163 def test_measurements_finite(self):
164 """Measured fluxes and centroids should always be finite.
165 """
166 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
168 # Set up the simulated images
169 noiseLevel = 1.
170 staticSeed = 1
171 transientSeed = 6
172 xSize = 256
173 ySize = 256
174 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
175 "xSize": xSize, "ySize": ySize}
176 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
177 nSrc=1, **kwargs)
178 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
179 nSrc=1, **kwargs)
180 rng = np.random.RandomState(3)
181 xLoc = np.arange(-5, xSize+5, 10)
182 rng.shuffle(xLoc)
183 yLoc = np.arange(-5, ySize+5, 10)
184 rng.shuffle(yLoc)
185 transients, transientSources = makeTestImage(seed=transientSeed,
186 nSrc=len(xLoc), fluxLevel=1000.,
187 noiseLevel=noiseLevel, noiseSeed=8,
188 xLoc=xLoc, yLoc=yLoc,
189 **kwargs)
190 difference = science.clone()
191 difference.maskedImage -= matchedTemplate.maskedImage
192 difference.maskedImage += transients.maskedImage
194 # Configure the detection Task
195 detectionTask = self._setup_detection(doForcedMeasurement=True)
197 # Run detection and check the results
198 output = detectionTask.run(science, matchedTemplate, difference)
200 for column in columnNames:
201 self._check_values(output.diaSources[column])
202 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
203 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
204 self._check_values(output.diaSources.getPsfInstFlux())
206 def test_raise_config_schema_mismatch(self):
207 """Check that sources with specified flags are removed from the catalog.
208 """
209 # Configure the detection Task, and and set a config that is not in the schema
210 with self.assertRaises(InvalidQuantumError):
211 self._setup_detection(badSourceFlags=["Bogus_flag_42"])
213 def test_remove_unphysical(self):
214 """Check that sources with specified flags are removed from the catalog.
215 """
216 # Set up the simulated images
217 noiseLevel = 1.
218 staticSeed = 1
219 xSize = 256
220 ySize = 256
221 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize}
222 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
223 nSrc=1, **kwargs)
224 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
225 nSrc=1, **kwargs)
226 difference = science.clone()
227 bbox = difference.getBBox()
228 difference.maskedImage -= matchedTemplate.maskedImage
230 # Configure the detection Task, and remove unphysical sources
231 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
232 badSourceFlags=["base_PixelFlags_flag_offimage", ])
234 # Run detection and check the results
235 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
236 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
237 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove)
238 # Verify that all sources are physical
239 self.assertEqual(nBadDoRemove, 0)
240 # Set a few centroids outside the image bounding box
241 nSetBad = 5
242 for src in diaSources[0: nSetBad]:
243 src["slot_Centroid_x"] += xSize
244 src["slot_Centroid_y"] += ySize
245 src["base_PixelFlags_flag_offimage"] = True
246 # Verify that these sources are outside the image
247 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY())
248 nBad = np.count_nonzero(badDiaSrc)
249 self.assertEqual(nBad, nSetBad)
250 diaSourcesNoBad = detectionTask._removeBadSources(diaSources)
251 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY())
253 # Verify that no sources outside the image bounding box remain
254 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0)
255 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad)
257 def test_detect_transients(self):
258 """Run detection on a difference image containing transients.
259 """
260 # Set up the simulated images
261 noiseLevel = 1.
262 staticSeed = 1
263 transientSeed = 6
264 fluxLevel = 500
265 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
266 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
267 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
269 # Configure the detection Task
270 detectionTask = self._setup_detection(doMerge=False)
271 kwargs["seed"] = transientSeed
272 kwargs["nSrc"] = 10
273 kwargs["fluxLevel"] = 1000
275 # Run detection and check the results
276 def _detection_wrapper(positive=True):
277 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
278 difference = science.clone()
279 difference.maskedImage -= matchedTemplate.maskedImage
280 if positive:
281 difference.maskedImage += transients.maskedImage
282 else:
283 difference.maskedImage -= transients.maskedImage
284 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
285 # science image if we've e.g. removed parents post-deblending.
286 # Pass a clone of the science image, so that it doesn't disrupt
287 # later tests.
288 output = detectionTask.run(science.clone(), matchedTemplate, difference)
289 refIds = []
290 scale = 1. if positive else -1.
291 for diaSource in output.diaSources:
292 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
293 _detection_wrapper(positive=True)
294 _detection_wrapper(positive=False)
296 def test_missing_mask_planes(self):
297 """Check that detection runs with missing mask planes.
298 """
299 # Set up the simulated images
300 noiseLevel = 1.
301 fluxLevel = 500
302 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []}
303 # Use different seeds for the science and template so every source is a diaSource
304 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
305 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
307 difference = science.clone()
308 difference.maskedImage -= matchedTemplate.maskedImage
309 detectionTask = self._setup_detection()
311 # Verify that detection runs without errors
312 detectionTask.run(science, matchedTemplate, difference)
314 def test_detect_dipoles(self):
315 """Run detection on a difference image containing dipoles.
316 """
317 # Set up the simulated images
318 noiseLevel = 1.
319 staticSeed = 1
320 fluxLevel = 1000
321 fluxRange = 1.5
322 nSources = 10
323 offset = 1
324 xSize = 300
325 ySize = 300
326 kernelSize = 32
327 # Avoid placing sources near the edge for this test, so that we can
328 # easily check that the correct number of sources are detected.
329 templateBorderSize = kernelSize//2
330 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
331 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
332 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
333 "xSize": xSize, "ySize": ySize}
334 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
335 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
336 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
337 difference = science.clone()
338 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
339 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
340 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
341 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
343 detectionTask = self._setup_detection(doMerge=True)
344 output = detectionTask.run(science, matchedTemplate, difference)
345 self.assertEqual(len(output.diaSources), len(sources))
346 refIds = []
347 for diaSource in output.diaSources:
348 if diaSource[dipoleFlag]:
349 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
350 rtol=0.05, atol=None, usePsfFlux=False)
351 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
352 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
353 else:
354 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
356 def test_sky_sources(self):
357 """Add sky sources and check that they are sufficiently far from other
358 sources and have negligible flux.
359 """
360 # Set up the simulated images
361 noiseLevel = 1.
362 staticSeed = 1
363 transientSeed = 6
364 transientFluxLevel = 1000.
365 transientFluxRange = 1.5
366 fluxLevel = 500
367 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
368 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
369 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
370 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
371 nSrc=10, fluxLevel=transientFluxLevel,
372 fluxRange=transientFluxRange,
373 noiseLevel=noiseLevel, noiseSeed=8)
374 difference = science.clone()
375 difference.maskedImage -= matchedTemplate.maskedImage
376 difference.maskedImage += transients.maskedImage
377 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2
379 # Configure the detection Task
380 detectionTask = self._setup_detection(doSkySources=True)
382 # Run detection and check the results
383 output = detectionTask.run(science, matchedTemplate, difference,
384 idFactory=self.idGenerator.make_table_id_factory())
385 skySources = output.diaSources[output.diaSources["sky_source"]]
386 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources)
387 for skySource in skySources:
388 # The sky sources should not be close to any other source
389 with self.assertRaises(AssertionError):
390 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
391 with self.assertRaises(AssertionError):
392 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
393 # The sky sources should have low flux levels.
394 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
395 atol=np.sqrt(transientFluxRange*transientFluxLevel))
397 # Catalog ids should be very large from this id generator.
398 self.assertTrue(all(output.diaSources['id'] > 1000000000))
400 def test_edge_detections(self):
401 """Sources with certain bad mask planes set should not be detected.
402 """
403 # Set up the simulated images
404 noiseLevel = 1.
405 staticSeed = 1
406 transientSeed = 6
407 fluxLevel = 500
408 radius = 2
409 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
410 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
411 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
413 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask
414 # Configure the detection Task
415 detectionTask = self._setup_detection()
416 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
417 nBad = len(excludeMaskPlanes)
418 self.assertGreater(nBad, 0)
419 kwargs["seed"] = transientSeed
420 kwargs["nSrc"] = nBad
421 kwargs["fluxLevel"] = 1000
423 # Run detection and check the results
424 def _detection_wrapper(setFlags=True):
425 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
426 difference = science.clone()
427 difference.maskedImage -= matchedTemplate.maskedImage
428 difference.maskedImage += transients.maskedImage
429 if setFlags:
430 for src, badMask in zip(transientSources, excludeMaskPlanes):
431 srcX = int(src.getX())
432 srcY = int(src.getY())
433 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
434 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
435 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
436 output = detectionTask.run(science, matchedTemplate, difference)
437 refIds = []
438 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes)
439 if setFlags:
440 self.assertEqual(np.sum(~goodSrcFlags), nBad)
441 else:
442 self.assertEqual(np.sum(~goodSrcFlags), 0)
443 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
444 if ~goodSrcFlag:
445 with self.assertRaises(AssertionError):
446 self._check_diaSource(transientSources, diaSource, refIds=refIds)
447 else:
448 self._check_diaSource(transientSources, diaSource, refIds=refIds)
449 _detection_wrapper(setFlags=False)
450 _detection_wrapper(setFlags=True)
452 def test_fake_mask_plane_propagation(self):
453 """Test that we have the mask planes related to fakes in diffim images.
454 This is testing method called updateMasks
455 """
456 xSize = 256
457 ySize = 256
458 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
459 science_fake_img, science_fake_sources = makeTestImage(
460 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1
461 )
462 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
463 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
464 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1
465 )
466 # created fakes and added them to the images
467 science.image += science_fake_img.image
468 template.image += tmplt_fake_img.image
470 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
471 # adding mask planes to both science and template images
472 science.mask.addMaskPlane("FAKE")
473 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE")
474 template.mask.addMaskPlane("FAKE")
475 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE")
477 # makeTestImage sets the DETECTED plane on the sources; we can use
478 # that to set the FAKE plane on the science and template images.
479 detected = science_fake_img.mask.getPlaneBitMask("DETECTED")
480 fake_pixels = (science_fake_img.mask.array & detected).nonzero()
481 science.mask.array[fake_pixels] |= science_fake_bitmask
482 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED")
483 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero()
484 template.mask.array[fake_pixels] |= science_fake_bitmask
486 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0
487 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0
489 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass()
490 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig)
491 subtraction = subtractTask.run(template, science, sources)
493 # check subtraction mask plane is set where we set the previous masks
494 diff_mask = subtraction.difference.mask
496 # science mask should be now in INJECTED
497 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
499 # template mask should be now in INJECTED_TEMPLATE
500 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
502 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int))
503 # The template is convolved, so the INJECTED_TEMPLATE mask plane may
504 # include more pixels than the FAKE mask plane
505 injTmplt_masked &= template_fake_masked
506 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int))
508 # Now check that detection of fakes have the correct flag for injections
509 detectionTask = self._setup_detection()
510 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
511 nBad = len(excludeMaskPlanes)
512 self.assertEqual(nBad, 1)
514 output = detectionTask.run(subtraction.matchedScience,
515 subtraction.matchedTemplate,
516 subtraction.difference)
518 sci_refIds = []
519 tmpl_refIds = []
520 for diaSrc in output.diaSources:
521 if diaSrc['base_PsfFlux_instFlux'] > 0:
522 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds)
523 self.assertTrue(diaSrc['base_PixelFlags_flag_injected'])
524 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter'])
525 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template'])
526 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
527 else:
528 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds)
529 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template'])
530 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
531 self.assertFalse(diaSrc['base_PixelFlags_flag_injected'])
532 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter'])
535class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
536 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask
538 def test_detection_xy0(self):
539 """Basic functionality test with non-zero x0 and y0.
540 """
541 # Set up the simulated images
542 noiseLevel = 1.
543 staticSeed = 1
544 fluxLevel = 500
545 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
546 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
547 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
548 difference = science.clone()
549 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
550 scienceKernel = science.psf.getKernel()
551 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
553 # Configure the detection Task
554 detectionTask = self._setup_detection()
556 # Run detection and check the results
557 output = detectionTask.run(science, matchedTemplate, difference, score,
558 idFactory=self.idGenerator.make_table_id_factory())
560 # Catalog ids should be very large from this id generator.
561 self.assertTrue(all(output.diaSources['id'] > 1000000000))
562 subtractedMeasuredExposure = output.subtractedMeasuredExposure
564 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
566 def test_measurements_finite(self):
567 """Measured fluxes and centroids should always be finite.
568 """
569 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
571 # Set up the simulated images
572 noiseLevel = 1.
573 staticSeed = 1
574 transientSeed = 6
575 xSize = 256
576 ySize = 256
577 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
578 "xSize": xSize, "ySize": ySize}
579 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
580 nSrc=1, **kwargs)
581 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
582 nSrc=1, **kwargs)
583 rng = np.random.RandomState(3)
584 xLoc = np.arange(-5, xSize+5, 10)
585 rng.shuffle(xLoc)
586 yLoc = np.arange(-5, ySize+5, 10)
587 rng.shuffle(yLoc)
588 transients, transientSources = makeTestImage(seed=transientSeed,
589 nSrc=len(xLoc), fluxLevel=1000.,
590 noiseLevel=noiseLevel, noiseSeed=8,
591 xLoc=xLoc, yLoc=yLoc,
592 **kwargs)
593 difference = science.clone()
594 difference.maskedImage -= matchedTemplate.maskedImage
595 difference.maskedImage += transients.maskedImage
596 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
597 scienceKernel = science.psf.getKernel()
598 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
600 # Configure the detection Task
601 detectionTask = self._setup_detection(doForcedMeasurement=True)
603 # Run detection and check the results
604 output = detectionTask.run(science, matchedTemplate, difference, score)
606 for column in columnNames:
607 self._check_values(output.diaSources[column])
608 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
609 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
610 self._check_values(output.diaSources.getPsfInstFlux())
612 def test_detect_transients(self):
613 """Run detection on a difference image containing transients.
614 """
615 # Set up the simulated images
616 noiseLevel = 1.
617 staticSeed = 1
618 transientSeed = 6
619 fluxLevel = 500
620 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
621 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
622 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
623 scienceKernel = science.psf.getKernel()
624 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
626 # Configure the detection Task
627 detectionTask = self._setup_detection(doMerge=False)
628 kwargs["seed"] = transientSeed
629 kwargs["nSrc"] = 10
630 kwargs["fluxLevel"] = 1000
632 # Run detection and check the results
633 def _detection_wrapper(positive=True):
634 """Simulate positive or negative transients and run detection.
636 Parameters
637 ----------
638 positive : `bool`, optional
639 If set, use positive transient sources.
640 """
642 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
643 difference = science.clone()
644 difference.maskedImage -= matchedTemplate.maskedImage
645 if positive:
646 difference.maskedImage += transients.maskedImage
647 else:
648 difference.maskedImage -= transients.maskedImage
649 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
650 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
651 # science image if we've e.g. removed parents post-deblending.
652 # Pass a clone of the science image, so that it doesn't disrupt
653 # later tests.
654 output = detectionTask.run(science.clone(), matchedTemplate, difference, score)
655 refIds = []
656 scale = 1. if positive else -1.
657 # sources near the edge may have untrustworthy centroids
658 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge']
659 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
660 if goodSrcFlag:
661 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
662 _detection_wrapper(positive=True)
663 _detection_wrapper(positive=False)
665 def test_detect_dipoles(self):
666 """Run detection on a difference image containing dipoles.
667 """
668 # Set up the simulated images
669 noiseLevel = 1.
670 staticSeed = 1
671 fluxLevel = 1000
672 fluxRange = 1.5
673 nSources = 10
674 offset = 1
675 xSize = 300
676 ySize = 300
677 kernelSize = 32
678 # Avoid placing sources near the edge for this test, so that we can
679 # easily check that the correct number of sources are detected.
680 templateBorderSize = kernelSize//2
681 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
682 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
683 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
684 "xSize": xSize, "ySize": ySize}
685 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
686 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
687 difference = science.clone()
688 # Shift the template by a pixel in order to make dipoles in the difference image.
689 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
690 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
691 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
692 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
693 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
694 scienceKernel = science.psf.getKernel()
695 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
697 detectionTask = self._setup_detection()
698 output = detectionTask.run(science, matchedTemplate, difference, score)
699 self.assertEqual(len(output.diaSources), len(sources))
700 refIds = []
701 for diaSource in output.diaSources:
702 if diaSource[dipoleFlag]:
703 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
704 rtol=0.05, atol=None, usePsfFlux=False)
705 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
706 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
707 else:
708 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
710 def test_sky_sources(self):
711 """Add sky sources and check that they are sufficiently far from other
712 sources and have negligible flux.
713 """
714 # Set up the simulated images
715 noiseLevel = 1.
716 staticSeed = 1
717 transientSeed = 6
718 transientFluxLevel = 1000.
719 transientFluxRange = 1.5
720 fluxLevel = 500
721 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
722 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
723 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
724 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
725 nSrc=10, fluxLevel=transientFluxLevel,
726 fluxRange=transientFluxRange,
727 noiseLevel=noiseLevel, noiseSeed=8)
728 difference = science.clone()
729 difference.maskedImage -= matchedTemplate.maskedImage
730 difference.maskedImage += transients.maskedImage
731 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
732 scienceKernel = science.psf.getKernel()
733 kernelWidth = np.max(scienceKernel.getDimensions())//2
734 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
736 # Configure the detection Task
737 detectionTask = self._setup_detection(doSkySources=True)
739 # Run detection and check the results
740 output = detectionTask.run(science, matchedTemplate, difference, score,
741 idFactory=self.idGenerator.make_table_id_factory())
742 nSkySourcesGenerated = detectionTask.metadata["nSkySources"]
743 skySources = output.diaSources[output.diaSources["sky_source"]]
744 self.assertEqual(len(skySources), nSkySourcesGenerated)
745 for skySource in skySources:
746 # The sky sources should not be close to any other source
747 with self.assertRaises(AssertionError):
748 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
749 with self.assertRaises(AssertionError):
750 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
751 # The sky sources should have low flux levels.
752 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
753 atol=np.sqrt(transientFluxRange*transientFluxLevel))
755 # Catalog ids should be very large from this id generator.
756 self.assertTrue(all(output.diaSources['id'] > 1000000000))
758 def test_edge_detections(self):
759 """Sources with certain bad mask planes set should not be detected.
760 """
761 # Set up the simulated images
762 noiseLevel = 1.
763 staticSeed = 1
764 transientSeed = 6
765 fluxLevel = 500
766 radius = 2
767 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
768 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
769 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
771 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
772 scienceKernel = science.psf.getKernel()
773 # Configure the detection Task
774 detectionTask = self._setup_detection()
775 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
776 nBad = len(excludeMaskPlanes)
777 self.assertGreater(nBad, 0)
778 kwargs["seed"] = transientSeed
779 kwargs["nSrc"] = nBad
780 kwargs["fluxLevel"] = 1000
782 # Run detection and check the results
783 def _detection_wrapper(setFlags=True):
784 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
785 difference = science.clone()
786 difference.maskedImage -= matchedTemplate.maskedImage
787 difference.maskedImage += transients.maskedImage
788 if setFlags:
789 for src, badMask in zip(transientSources, excludeMaskPlanes):
790 srcX = int(src.getX())
791 srcY = int(src.getY())
792 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
793 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
794 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
795 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
796 output = detectionTask.run(science, matchedTemplate, difference, score)
797 refIds = []
798 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes)
799 if setFlags:
800 self.assertEqual(np.sum(~goodSrcFlags), nBad)
801 else:
802 self.assertEqual(np.sum(~goodSrcFlags), 0)
803 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
804 if ~goodSrcFlag:
805 with self.assertRaises(AssertionError):
806 self._check_diaSource(transientSources, diaSource, refIds=refIds)
807 else:
808 self._check_diaSource(transientSources, diaSource, refIds=refIds)
809 _detection_wrapper(setFlags=False)
810 _detection_wrapper(setFlags=True)
813def setup_module(module):
814 lsst.utils.tests.init()
817class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
818 pass
821if __name__ == "__main__": 821 ↛ 822line 821 didn't jump to line 822, because the condition on line 821 was never true
822 lsst.utils.tests.init()
823 unittest.main()