Coverage for tests/test_detectAndMeasure.py: 6%
487 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-11 03:14 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-04-11 03:14 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import unittest
25import lsst.geom
26from lsst.ip.diffim import detectAndMeasure, subtractImages
27from lsst.ip.diffim.utils import makeTestImage
28from lsst.pipe.base import InvalidQuantumError
29import lsst.utils.tests
32class DetectAndMeasureTestBase:
34 def _check_diaSource(self, refSources, diaSource, refIds=None,
35 matchDistance=1., scale=1., usePsfFlux=True,
36 rtol=0.025, atol=None):
37 """Match a diaSource with a source in a reference catalog
38 and compare properties.
40 Parameters
41 ----------
42 refSources : `lsst.afw.table.SourceCatalog`
43 The reference catalog.
44 diaSource : `lsst.afw.table.SourceRecord`
45 The new diaSource to match to the reference catalog.
46 refIds : `list` of `int`, optional
47 Source IDs of previously associated diaSources.
48 matchDistance : `float`, optional
49 Maximum distance allowed between the detected and reference source
50 locations, in pixels.
51 scale : `float`, optional
52 Optional factor to scale the flux by before performing the test.
53 usePsfFlux : `bool`, optional
54 If set, test the PsfInstFlux field, otherwise use ApInstFlux.
55 rtol : `float`, optional
56 Relative tolerance of the flux value test.
57 atol : `float`, optional
58 Absolute tolerance of the flux value test.
59 """
60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2
61 + (diaSource.getY() - refSources.getY())**2)
62 self.assertLess(min(distance), matchDistance)
63 src = refSources[np.argmin(distance)]
64 if refIds is not None:
65 # Check that the same source was not previously associated
66 self.assertNotIn(src.getId(), refIds)
67 refIds.append(src.getId())
68 if atol is None:
69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux()
70 if usePsfFlux:
71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(),
72 rtol=rtol, atol=atol)
73 else:
74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(),
75 rtol=rtol, atol=atol)
77 def _check_values(self, values, minValue=None, maxValue=None):
78 """Verify that an array has finite values, and optionally that they are
79 within specified minimum and maximum bounds.
81 Parameters
82 ----------
83 values : `numpy.ndarray`
84 Array of values to check.
85 minValue : `float`, optional
86 Minimum allowable value.
87 maxValue : `float`, optional
88 Maximum allowable value.
89 """
90 self.assertTrue(np.all(np.isfinite(values)))
91 if minValue is not None:
92 self.assertTrue(np.all(values >= minValue))
93 if maxValue is not None:
94 self.assertTrue(np.all(values <= maxValue))
96 def _setup_detection(self, doSkySources=False, nSkySources=5, doWriteMetrics=False, **kwargs):
97 """Setup and configure the detection and measurement PipelineTask.
99 Parameters
100 ----------
101 doSkySources : `bool`, optional
102 Generate sky sources.
103 nSkySources : `int`, optional
104 The number of sky sources to add in isolated background regions.
105 **kwargs
106 Any additional config parameters to set.
108 Returns
109 -------
110 `lsst.pipe.base.PipelineTask`
111 The configured Task to use for detection and measurement.
112 """
113 config = self.detectionTask.ConfigClass()
114 config.doSkySources = doSkySources
115 if doSkySources:
116 config.skySources.nSources = nSkySources
117 config.doWriteMetrics = doWriteMetrics
118 config.update(**kwargs)
120 # Make a realistic id generator so that output catalog ids are useful.
121 dataId = lsst.daf.butler.DataCoordinate.standardize(
122 instrument="I",
123 visit=42,
124 detector=12,
125 universe=lsst.daf.butler.DimensionUniverse(),
126 )
127 config.idGenerator.packer.name = "observation"
128 config.idGenerator.packer["observation"].n_observations = 10000
129 config.idGenerator.packer["observation"].n_detectors = 99
130 config.idGenerator.n_releases = 8
131 config.idGenerator.release_id = 2
132 self.idGenerator = config.idGenerator.apply(dataId)
134 return self.detectionTask(config=config)
137class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
138 detectionTask = detectAndMeasure.DetectAndMeasureTask
140 def test_detection_xy0(self):
141 """Basic functionality test with non-zero x0 and y0.
142 """
143 # Set up the simulated images
144 noiseLevel = 1.
145 staticSeed = 1
146 fluxLevel = 500
147 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
148 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
149 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
150 difference = science.clone()
152 # Configure the detection Task
153 detectionTask = self._setup_detection()
155 # Run detection and check the results
156 output = detectionTask.run(science, matchedTemplate, difference,
157 idFactory=self.idGenerator.make_table_id_factory())
158 subtractedMeasuredExposure = output.subtractedMeasuredExposure
160 # Catalog ids should be very large from this id generator.
161 self.assertTrue(all(output.diaSources['id'] > 1000000000))
162 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
164 def test_measurements_finite(self):
165 """Measured fluxes and centroids should always be finite.
166 """
167 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
169 # Set up the simulated images
170 noiseLevel = 1.
171 staticSeed = 1
172 transientSeed = 6
173 xSize = 256
174 ySize = 256
175 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
176 "xSize": xSize, "ySize": ySize}
177 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
178 nSrc=1, **kwargs)
179 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
180 nSrc=1, **kwargs)
181 rng = np.random.RandomState(3)
182 xLoc = np.arange(-5, xSize+5, 10)
183 rng.shuffle(xLoc)
184 yLoc = np.arange(-5, ySize+5, 10)
185 rng.shuffle(yLoc)
186 transients, transientSources = makeTestImage(seed=transientSeed,
187 nSrc=len(xLoc), fluxLevel=1000.,
188 noiseLevel=noiseLevel, noiseSeed=8,
189 xLoc=xLoc, yLoc=yLoc,
190 **kwargs)
191 difference = science.clone()
192 difference.maskedImage -= matchedTemplate.maskedImage
193 difference.maskedImage += transients.maskedImage
195 # Configure the detection Task
196 detectionTask = self._setup_detection(doForcedMeasurement=True)
198 # Run detection and check the results
199 output = detectionTask.run(science, matchedTemplate, difference)
201 for column in columnNames:
202 self._check_values(output.diaSources[column])
203 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
204 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
205 self._check_values(output.diaSources.getPsfInstFlux())
207 def test_raise_config_schema_mismatch(self):
208 """Check that sources with specified flags are removed from the catalog.
209 """
210 # Configure the detection Task, and and set a config that is not in the schema
211 with self.assertRaises(InvalidQuantumError):
212 self._setup_detection(badSourceFlags=["Bogus_flag_42"])
214 def test_remove_unphysical(self):
215 """Check that sources with specified flags are removed from the catalog.
216 """
217 # Set up the simulated images
218 noiseLevel = 1.
219 staticSeed = 1
220 xSize = 256
221 ySize = 256
222 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize}
223 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
224 nSrc=1, **kwargs)
225 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
226 nSrc=1, **kwargs)
227 difference = science.clone()
228 bbox = difference.getBBox()
229 difference.maskedImage -= matchedTemplate.maskedImage
231 # Configure the detection Task, and remove unphysical sources
232 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
233 badSourceFlags=["base_PixelFlags_flag_offimage", ])
235 # Run detection and check the results
236 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
237 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
238 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove)
239 # Verify that all sources are physical
240 self.assertEqual(nBadDoRemove, 0)
241 # Set a few centroids outside the image bounding box
242 nSetBad = 5
243 for src in diaSources[0: nSetBad]:
244 src["slot_Centroid_x"] += xSize
245 src["slot_Centroid_y"] += ySize
246 src["base_PixelFlags_flag_offimage"] = True
247 # Verify that these sources are outside the image
248 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY())
249 nBad = np.count_nonzero(badDiaSrc)
250 self.assertEqual(nBad, nSetBad)
251 diaSourcesNoBad = detectionTask._removeBadSources(diaSources)
252 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY())
254 # Verify that no sources outside the image bounding box remain
255 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0)
256 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad)
258 def test_detect_transients(self):
259 """Run detection on a difference image containing transients.
260 """
261 # Set up the simulated images
262 noiseLevel = 1.
263 staticSeed = 1
264 transientSeed = 6
265 fluxLevel = 500
266 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
267 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
268 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
270 # Configure the detection Task
271 detectionTask = self._setup_detection(doMerge=False)
272 kwargs["seed"] = transientSeed
273 kwargs["nSrc"] = 10
274 kwargs["fluxLevel"] = 1000
276 # Run detection and check the results
277 def _detection_wrapper(positive=True):
278 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
279 difference = science.clone()
280 difference.maskedImage -= matchedTemplate.maskedImage
281 if positive:
282 difference.maskedImage += transients.maskedImage
283 else:
284 difference.maskedImage -= transients.maskedImage
285 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
286 # science image if we've e.g. removed parents post-deblending.
287 # Pass a clone of the science image, so that it doesn't disrupt
288 # later tests.
289 output = detectionTask.run(science.clone(), matchedTemplate, difference)
290 refIds = []
291 scale = 1. if positive else -1.
292 for diaSource in output.diaSources:
293 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
294 _detection_wrapper(positive=True)
295 _detection_wrapper(positive=False)
297 def test_missing_mask_planes(self):
298 """Check that detection runs with missing mask planes.
299 """
300 # Set up the simulated images
301 noiseLevel = 1.
302 fluxLevel = 500
303 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []}
304 # Use different seeds for the science and template so every source is a diaSource
305 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
306 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
308 difference = science.clone()
309 difference.maskedImage -= matchedTemplate.maskedImage
310 detectionTask = self._setup_detection()
312 # Verify that detection runs without errors
313 detectionTask.run(science, matchedTemplate, difference)
315 def test_detect_dipoles(self):
316 """Run detection on a difference image containing dipoles.
317 """
318 # Set up the simulated images
319 noiseLevel = 1.
320 staticSeed = 1
321 fluxLevel = 1000
322 fluxRange = 1.5
323 nSources = 10
324 offset = 1
325 xSize = 300
326 ySize = 300
327 kernelSize = 32
328 # Avoid placing sources near the edge for this test, so that we can
329 # easily check that the correct number of sources are detected.
330 templateBorderSize = kernelSize//2
331 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
332 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
333 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
334 "xSize": xSize, "ySize": ySize}
335 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
336 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
337 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
338 difference = science.clone()
339 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
340 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
341 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
342 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
344 detectionTask = self._setup_detection(doMerge=True)
345 output = detectionTask.run(science, matchedTemplate, difference)
346 self.assertEqual(len(output.diaSources), len(sources))
347 refIds = []
348 for diaSource in output.diaSources:
349 if diaSource[dipoleFlag]:
350 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
351 rtol=0.05, atol=None, usePsfFlux=False)
352 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
353 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
354 else:
355 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
357 def test_sky_sources(self):
358 """Add sky sources and check that they are sufficiently far from other
359 sources and have negligible flux.
360 """
361 # Set up the simulated images
362 noiseLevel = 1.
363 staticSeed = 1
364 transientSeed = 6
365 transientFluxLevel = 1000.
366 transientFluxRange = 1.5
367 fluxLevel = 500
368 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
369 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
370 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
371 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
372 nSrc=10, fluxLevel=transientFluxLevel,
373 fluxRange=transientFluxRange,
374 noiseLevel=noiseLevel, noiseSeed=8)
375 difference = science.clone()
376 difference.maskedImage -= matchedTemplate.maskedImage
377 difference.maskedImage += transients.maskedImage
378 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2
380 # Configure the detection Task
381 detectionTask = self._setup_detection(doSkySources=True)
383 # Run detection and check the results
384 output = detectionTask.run(science, matchedTemplate, difference,
385 idFactory=self.idGenerator.make_table_id_factory())
386 skySources = output.diaSources[output.diaSources["sky_source"]]
387 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources)
388 for skySource in skySources:
389 # The sky sources should not be close to any other source
390 with self.assertRaises(AssertionError):
391 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
392 with self.assertRaises(AssertionError):
393 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
394 # The sky sources should have low flux levels.
395 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
396 atol=np.sqrt(transientFluxRange*transientFluxLevel))
398 # Catalog ids should be very large from this id generator.
399 self.assertTrue(all(output.diaSources['id'] > 1000000000))
401 def test_edge_detections(self):
402 """Sources with certain bad mask planes set should not be detected.
403 """
404 # Set up the simulated images
405 noiseLevel = 1.
406 staticSeed = 1
407 transientSeed = 6
408 fluxLevel = 500
409 radius = 2
410 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
411 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
412 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
414 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask
415 # Configure the detection Task
416 detectionTask = self._setup_detection()
417 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
418 nBad = len(excludeMaskPlanes)
419 self.assertGreater(nBad, 0)
420 kwargs["seed"] = transientSeed
421 kwargs["nSrc"] = nBad
422 kwargs["fluxLevel"] = 1000
424 # Run detection and check the results
425 def _detection_wrapper(setFlags=True):
426 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
427 difference = science.clone()
428 difference.maskedImage -= matchedTemplate.maskedImage
429 difference.maskedImage += transients.maskedImage
430 if setFlags:
431 for src, badMask in zip(transientSources, excludeMaskPlanes):
432 srcX = int(src.getX())
433 srcY = int(src.getY())
434 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
435 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
436 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
437 output = detectionTask.run(science, matchedTemplate, difference)
438 refIds = []
439 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes)
440 if setFlags:
441 self.assertEqual(np.sum(~goodSrcFlags), nBad)
442 else:
443 self.assertEqual(np.sum(~goodSrcFlags), 0)
444 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
445 if ~goodSrcFlag:
446 with self.assertRaises(AssertionError):
447 self._check_diaSource(transientSources, diaSource, refIds=refIds)
448 else:
449 self._check_diaSource(transientSources, diaSource, refIds=refIds)
450 _detection_wrapper(setFlags=False)
451 _detection_wrapper(setFlags=True)
453 def test_fake_mask_plane_propagation(self):
454 """Test that we have the mask planes related to fakes in diffim images.
455 This is testing method called updateMasks
456 """
457 xSize = 256
458 ySize = 256
459 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
460 science_fake_img, science_fake_sources = makeTestImage(
461 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1
462 )
463 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
464 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
465 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1
466 )
467 # created fakes and added them to the images
468 science.image += science_fake_img.image
469 template.image += tmplt_fake_img.image
471 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
472 # adding mask planes to both science and template images
473 science.mask.addMaskPlane("FAKE")
474 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE")
475 template.mask.addMaskPlane("FAKE")
476 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE")
478 # makeTestImage sets the DETECTED plane on the sources; we can use
479 # that to set the FAKE plane on the science and template images.
480 detected = science_fake_img.mask.getPlaneBitMask("DETECTED")
481 fake_pixels = (science_fake_img.mask.array & detected).nonzero()
482 science.mask.array[fake_pixels] |= science_fake_bitmask
483 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED")
484 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero()
485 template.mask.array[fake_pixels] |= science_fake_bitmask
487 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0
488 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0
490 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass()
491 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig)
492 subtraction = subtractTask.run(template, science, sources)
494 # check subtraction mask plane is set where we set the previous masks
495 diff_mask = subtraction.difference.mask
497 # science mask should be now in INJECTED
498 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
500 # template mask should be now in INJECTED_TEMPLATE
501 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
503 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int))
504 # The template is convolved, so the INJECTED_TEMPLATE mask plane may
505 # include more pixels than the FAKE mask plane
506 injTmplt_masked &= template_fake_masked
507 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int))
509 # Now check that detection of fakes have the correct flag for injections
510 detectionTask = self._setup_detection()
511 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
512 nBad = len(excludeMaskPlanes)
513 self.assertEqual(nBad, 1)
515 output = detectionTask.run(subtraction.matchedScience,
516 subtraction.matchedTemplate,
517 subtraction.difference)
519 sci_refIds = []
520 tmpl_refIds = []
521 for diaSrc in output.diaSources:
522 if diaSrc['base_PsfFlux_instFlux'] > 0:
523 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds)
524 self.assertTrue(diaSrc['base_PixelFlags_flag_injected'])
525 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter'])
526 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template'])
527 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
528 else:
529 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds)
530 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template'])
531 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
532 self.assertFalse(diaSrc['base_PixelFlags_flag_injected'])
533 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter'])
536class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
537 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask
539 def test_detection_xy0(self):
540 """Basic functionality test with non-zero x0 and y0.
541 """
542 # Set up the simulated images
543 noiseLevel = 1.
544 staticSeed = 1
545 fluxLevel = 500
546 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
547 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
548 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
549 difference = science.clone()
550 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
551 scienceKernel = science.psf.getKernel()
552 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
554 # Configure the detection Task
555 detectionTask = self._setup_detection()
557 # Run detection and check the results
558 output = detectionTask.run(science, matchedTemplate, difference, score,
559 idFactory=self.idGenerator.make_table_id_factory())
561 # Catalog ids should be very large from this id generator.
562 self.assertTrue(all(output.diaSources['id'] > 1000000000))
563 subtractedMeasuredExposure = output.subtractedMeasuredExposure
565 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
567 def test_measurements_finite(self):
568 """Measured fluxes and centroids should always be finite.
569 """
570 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
572 # Set up the simulated images
573 noiseLevel = 1.
574 staticSeed = 1
575 transientSeed = 6
576 xSize = 256
577 ySize = 256
578 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
579 "xSize": xSize, "ySize": ySize}
580 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
581 nSrc=1, **kwargs)
582 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
583 nSrc=1, **kwargs)
584 rng = np.random.RandomState(3)
585 xLoc = np.arange(-5, xSize+5, 10)
586 rng.shuffle(xLoc)
587 yLoc = np.arange(-5, ySize+5, 10)
588 rng.shuffle(yLoc)
589 transients, transientSources = makeTestImage(seed=transientSeed,
590 nSrc=len(xLoc), fluxLevel=1000.,
591 noiseLevel=noiseLevel, noiseSeed=8,
592 xLoc=xLoc, yLoc=yLoc,
593 **kwargs)
594 difference = science.clone()
595 difference.maskedImage -= matchedTemplate.maskedImage
596 difference.maskedImage += transients.maskedImage
597 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
598 scienceKernel = science.psf.getKernel()
599 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
601 # Configure the detection Task
602 detectionTask = self._setup_detection(doForcedMeasurement=True)
604 # Run detection and check the results
605 output = detectionTask.run(science, matchedTemplate, difference, score)
607 for column in columnNames:
608 self._check_values(output.diaSources[column])
609 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
610 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
611 self._check_values(output.diaSources.getPsfInstFlux())
613 def test_detect_transients(self):
614 """Run detection on a difference image containing transients.
615 """
616 # Set up the simulated images
617 noiseLevel = 1.
618 staticSeed = 1
619 transientSeed = 6
620 fluxLevel = 500
621 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
622 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
623 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
624 scienceKernel = science.psf.getKernel()
625 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
627 # Configure the detection Task
628 detectionTask = self._setup_detection(doMerge=False)
629 kwargs["seed"] = transientSeed
630 kwargs["nSrc"] = 10
631 kwargs["fluxLevel"] = 1000
633 # Run detection and check the results
634 def _detection_wrapper(positive=True):
635 """Simulate positive or negative transients and run detection.
637 Parameters
638 ----------
639 positive : `bool`, optional
640 If set, use positive transient sources.
641 """
643 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
644 difference = science.clone()
645 difference.maskedImage -= matchedTemplate.maskedImage
646 if positive:
647 difference.maskedImage += transients.maskedImage
648 else:
649 difference.maskedImage -= transients.maskedImage
650 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
651 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
652 # science image if we've e.g. removed parents post-deblending.
653 # Pass a clone of the science image, so that it doesn't disrupt
654 # later tests.
655 output = detectionTask.run(science.clone(), matchedTemplate, difference, score)
656 refIds = []
657 scale = 1. if positive else -1.
658 # sources near the edge may have untrustworthy centroids
659 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge']
660 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
661 if goodSrcFlag:
662 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
663 _detection_wrapper(positive=True)
664 _detection_wrapper(positive=False)
666 def test_detect_dipoles(self):
667 """Run detection on a difference image containing dipoles.
668 """
669 # Set up the simulated images
670 noiseLevel = 1.
671 staticSeed = 1
672 fluxLevel = 1000
673 fluxRange = 1.5
674 nSources = 10
675 offset = 1
676 xSize = 300
677 ySize = 300
678 kernelSize = 32
679 # Avoid placing sources near the edge for this test, so that we can
680 # easily check that the correct number of sources are detected.
681 templateBorderSize = kernelSize//2
682 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
683 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
684 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
685 "xSize": xSize, "ySize": ySize}
686 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
687 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
688 difference = science.clone()
689 # Shift the template by a pixel in order to make dipoles in the difference image.
690 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
691 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
692 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
693 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
694 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
695 scienceKernel = science.psf.getKernel()
696 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
698 detectionTask = self._setup_detection()
699 output = detectionTask.run(science, matchedTemplate, difference, score)
700 self.assertEqual(len(output.diaSources), len(sources))
701 refIds = []
702 for diaSource in output.diaSources:
703 if diaSource[dipoleFlag]:
704 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
705 rtol=0.05, atol=None, usePsfFlux=False)
706 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
707 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
708 else:
709 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
711 def test_sky_sources(self):
712 """Add sky sources and check that they are sufficiently far from other
713 sources and have negligible flux.
714 """
715 # Set up the simulated images
716 noiseLevel = 1.
717 staticSeed = 1
718 transientSeed = 6
719 transientFluxLevel = 1000.
720 transientFluxRange = 1.5
721 fluxLevel = 500
722 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
723 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
724 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
725 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
726 nSrc=10, fluxLevel=transientFluxLevel,
727 fluxRange=transientFluxRange,
728 noiseLevel=noiseLevel, noiseSeed=8)
729 difference = science.clone()
730 difference.maskedImage -= matchedTemplate.maskedImage
731 difference.maskedImage += transients.maskedImage
732 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
733 scienceKernel = science.psf.getKernel()
734 kernelWidth = np.max(scienceKernel.getDimensions())//2
735 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
737 # Configure the detection Task
738 detectionTask = self._setup_detection(doSkySources=True)
740 # Run detection and check the results
741 output = detectionTask.run(science, matchedTemplate, difference, score,
742 idFactory=self.idGenerator.make_table_id_factory())
743 nSkySourcesGenerated = detectionTask.metadata["n_skySources"]
744 skySources = output.diaSources[output.diaSources["sky_source"]]
745 self.assertEqual(len(skySources), nSkySourcesGenerated)
746 for skySource in skySources:
747 # The sky sources should not be close to any other source
748 with self.assertRaises(AssertionError):
749 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
750 with self.assertRaises(AssertionError):
751 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
752 # The sky sources should have low flux levels.
753 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
754 atol=np.sqrt(transientFluxRange*transientFluxLevel))
756 # Catalog ids should be very large from this id generator.
757 self.assertTrue(all(output.diaSources['id'] > 1000000000))
759 def test_edge_detections(self):
760 """Sources with certain bad mask planes set should not be detected.
761 """
762 # Set up the simulated images
763 noiseLevel = 1.
764 staticSeed = 1
765 transientSeed = 6
766 fluxLevel = 500
767 radius = 2
768 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
769 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
770 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
772 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
773 scienceKernel = science.psf.getKernel()
774 # Configure the detection Task
775 detectionTask = self._setup_detection()
776 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
777 nBad = len(excludeMaskPlanes)
778 self.assertGreater(nBad, 0)
779 kwargs["seed"] = transientSeed
780 kwargs["nSrc"] = nBad
781 kwargs["fluxLevel"] = 1000
783 # Run detection and check the results
784 def _detection_wrapper(setFlags=True):
785 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
786 difference = science.clone()
787 difference.maskedImage -= matchedTemplate.maskedImage
788 difference.maskedImage += transients.maskedImage
789 if setFlags:
790 for src, badMask in zip(transientSources, excludeMaskPlanes):
791 srcX = int(src.getX())
792 srcY = int(src.getY())
793 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
794 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
795 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
796 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
797 output = detectionTask.run(science, matchedTemplate, difference, score)
798 refIds = []
799 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes)
800 if setFlags:
801 self.assertEqual(np.sum(~goodSrcFlags), nBad)
802 else:
803 self.assertEqual(np.sum(~goodSrcFlags), 0)
804 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
805 if ~goodSrcFlag:
806 with self.assertRaises(AssertionError):
807 self._check_diaSource(transientSources, diaSource, refIds=refIds)
808 else:
809 self._check_diaSource(transientSources, diaSource, refIds=refIds)
810 _detection_wrapper(setFlags=False)
811 _detection_wrapper(setFlags=True)
814def setup_module(module):
815 lsst.utils.tests.init()
818class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
819 pass
822if __name__ == "__main__": 822 ↛ 823line 822 didn't jump to line 823, because the condition on line 822 was never true
823 lsst.utils.tests.init()
824 unittest.main()