Coverage for tests/test_detectAndMeasure.py: 6%
509 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-26 03:37 -0700
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-26 03:37 -0700
1# This file is part of ip_diffim.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import numpy as np
23import unittest
25import lsst.geom
26from lsst.ip.diffim import detectAndMeasure, subtractImages
27from lsst.ip.diffim.utils import makeTestImage
28from lsst.pipe.base import InvalidQuantumError
29import lsst.utils.tests
32class DetectAndMeasureTestBase:
34 def _check_diaSource(self, refSources, diaSource, refIds=None,
35 matchDistance=1., scale=1., usePsfFlux=True,
36 rtol=0.021, atol=None):
37 """Match a diaSource with a source in a reference catalog
38 and compare properties.
40 Parameters
41 ----------
42 refSources : `lsst.afw.table.SourceCatalog`
43 The reference catalog.
44 diaSource : `lsst.afw.table.SourceRecord`
45 The new diaSource to match to the reference catalog.
46 refIds : `list` of `int`, optional
47 Source IDs of previously associated diaSources.
48 matchDistance : `float`, optional
49 Maximum distance allowed between the detected and reference source
50 locations, in pixels.
51 scale : `float`, optional
52 Optional factor to scale the flux by before performing the test.
53 usePsfFlux : `bool`, optional
54 If set, test the PsfInstFlux field, otherwise use ApInstFlux.
55 rtol : `float`, optional
56 Relative tolerance of the flux value test.
57 atol : `float`, optional
58 Absolute tolerance of the flux value test.
59 """
60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2
61 + (diaSource.getY() - refSources.getY())**2)
62 self.assertLess(min(distance), matchDistance)
63 src = refSources[np.argmin(distance)]
64 if refIds is not None:
65 # Check that the same source was not previously associated
66 self.assertNotIn(src.getId(), refIds)
67 refIds.append(src.getId())
68 if atol is None:
69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux()
70 if usePsfFlux:
71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(),
72 rtol=rtol, atol=atol)
73 else:
74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(),
75 rtol=rtol, atol=atol)
77 def _check_values(self, values, minValue=None, maxValue=None):
78 """Verify that an array has finite values, and optionally that they are
79 within specified minimum and maximum bounds.
81 Parameters
82 ----------
83 values : `numpy.ndarray`
84 Array of values to check.
85 minValue : `float`, optional
86 Minimum allowable value.
87 maxValue : `float`, optional
88 Maximum allowable value.
89 """
90 self.assertTrue(np.all(np.isfinite(values)))
91 if minValue is not None:
92 self.assertTrue(np.all(values >= minValue))
93 if maxValue is not None:
94 self.assertTrue(np.all(values <= maxValue))
96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs):
97 """Setup and configure the detection and measurement PipelineTask.
99 Parameters
100 ----------
101 doSkySources : `bool`, optional
102 Generate sky sources.
103 nSkySources : `int`, optional
104 The number of sky sources to add in isolated background regions.
105 **kwargs
106 Any additional config parameters to set.
108 Returns
109 -------
110 `lsst.pipe.base.PipelineTask`
111 The configured Task to use for detection and measurement.
112 """
113 config = self.detectionTask.ConfigClass()
114 config.doSkySources = doSkySources
115 if doSkySources:
116 config.skySources.nSources = nSkySources
117 config.update(**kwargs)
119 # Make a realistic id generator so that output catalog ids are useful.
120 dataId = lsst.daf.butler.DataCoordinate.standardize(
121 instrument="I",
122 visit=42,
123 detector=12,
124 universe=lsst.daf.butler.DimensionUniverse(),
125 )
126 config.idGenerator.packer.name = "observation"
127 config.idGenerator.packer["observation"].n_observations = 10000
128 config.idGenerator.packer["observation"].n_detectors = 99
129 config.idGenerator.n_releases = 8
130 config.idGenerator.release_id = 2
131 self.idGenerator = config.idGenerator.apply(dataId)
133 return self.detectionTask(config=config)
136class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
137 detectionTask = detectAndMeasure.DetectAndMeasureTask
139 def test_detection_xy0(self):
140 """Basic functionality test with non-zero x0 and y0.
141 """
142 # Set up the simulated images
143 noiseLevel = 1.
144 staticSeed = 1
145 fluxLevel = 500
146 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
147 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
148 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
149 difference = science.clone()
151 # Configure the detection Task
152 detectionTask = self._setup_detection()
154 # Run detection and check the results
155 output = detectionTask.run(science, matchedTemplate, difference,
156 idFactory=self.idGenerator.make_table_id_factory())
157 subtractedMeasuredExposure = output.subtractedMeasuredExposure
159 # Catalog ids should be very large from this id generator.
160 self.assertTrue(all(output.diaSources['id'] > 1000000000))
161 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
163 def test_measurements_finite(self):
164 """Measured fluxes and centroids should always be finite.
165 """
166 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
168 # Set up the simulated images
169 noiseLevel = 1.
170 staticSeed = 1
171 transientSeed = 6
172 xSize = 256
173 ySize = 256
174 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
175 "xSize": xSize, "ySize": ySize}
176 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
177 nSrc=1, **kwargs)
178 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
179 nSrc=1, **kwargs)
180 rng = np.random.RandomState(3)
181 xLoc = np.arange(-5, xSize+5, 10)
182 rng.shuffle(xLoc)
183 yLoc = np.arange(-5, ySize+5, 10)
184 rng.shuffle(yLoc)
185 transients, transientSources = makeTestImage(seed=transientSeed,
186 nSrc=len(xLoc), fluxLevel=1000.,
187 noiseLevel=noiseLevel, noiseSeed=8,
188 xLoc=xLoc, yLoc=yLoc,
189 **kwargs)
190 difference = science.clone()
191 difference.maskedImage -= matchedTemplate.maskedImage
192 difference.maskedImage += transients.maskedImage
194 # Configure the detection Task
195 detectionTask = self._setup_detection(doForcedMeasurement=True)
197 # Run detection and check the results
198 output = detectionTask.run(science, matchedTemplate, difference)
200 for column in columnNames:
201 self._check_values(output.diaSources[column])
202 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
203 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
204 self._check_values(output.diaSources.getPsfInstFlux())
206 def test_raise_config_schema_mismatch(self):
207 """Check that sources with specified flags are removed from the catalog.
208 """
209 # Configure the detection Task, and and set a config that is not in the schema
210 with self.assertRaises(InvalidQuantumError):
211 self._setup_detection(badSourceFlags=["Bogus_flag_42"])
213 def test_remove_unphysical(self):
214 """Check that sources with specified flags are removed from the catalog.
215 """
216 # Set up the simulated images
217 noiseLevel = 1.
218 staticSeed = 1
219 xSize = 256
220 ySize = 256
221 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize}
222 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
223 nSrc=1, **kwargs)
224 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
225 nSrc=1, **kwargs)
226 difference = science.clone()
227 bbox = difference.getBBox()
228 difference.maskedImage -= matchedTemplate.maskedImage
230 # Configure the detection Task, and do not remove unphysical sources
231 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
232 badSourceFlags=[])
234 # Run detection and check the results
235 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
236 badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
237 nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove)
238 # Verify that unphysical sources exist
239 self.assertGreater(nBadNoRemove, 0)
241 # Configure the detection Task, and remove unphysical sources
242 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20,
243 badSourceFlags=["base_PixelFlags_flag_offimage", ])
245 # Run detection and check the results
246 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources
247 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY())
248 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove)
249 # Verify that all sources are physical
250 self.assertEqual(nBadDoRemove, 0)
251 # Set a few centroids outside the image bounding box
252 nSetBad = 5
253 for src in diaSources[0: nSetBad]:
254 src["slot_Centroid_x"] += xSize
255 src["slot_Centroid_y"] += ySize
256 src["base_PixelFlags_flag_offimage"] = True
257 # Verify that these sources are outside the image
258 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY())
259 nBad = np.count_nonzero(badDiaSrc)
260 self.assertEqual(nBad, nSetBad)
261 diaSourcesNoBad = detectionTask._removeBadSources(diaSources)
262 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY())
264 # Verify that no sources outside the image bounding box remain
265 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0)
266 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad)
268 def test_detect_transients(self):
269 """Run detection on a difference image containing transients.
270 """
271 # Set up the simulated images
272 noiseLevel = 1.
273 staticSeed = 1
274 transientSeed = 6
275 fluxLevel = 500
276 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
277 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
278 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
280 # Configure the detection Task
281 detectionTask = self._setup_detection(doMerge=False)
282 kwargs["seed"] = transientSeed
283 kwargs["nSrc"] = 10
284 kwargs["fluxLevel"] = 1000
286 # Run detection and check the results
287 def _detection_wrapper(positive=True):
288 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
289 difference = science.clone()
290 difference.maskedImage -= matchedTemplate.maskedImage
291 if positive:
292 difference.maskedImage += transients.maskedImage
293 else:
294 difference.maskedImage -= transients.maskedImage
295 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
296 # science image if we've e.g. removed parents post-deblending.
297 # Pass a clone of the science image, so that it doesn't disrupt
298 # later tests.
299 output = detectionTask.run(science.clone(), matchedTemplate, difference)
300 refIds = []
301 scale = 1. if positive else -1.
302 for diaSource in output.diaSources:
303 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
304 _detection_wrapper(positive=True)
305 _detection_wrapper(positive=False)
307 def test_missing_mask_planes(self):
308 """Check that detection runs with missing mask planes.
309 """
310 # Set up the simulated images
311 noiseLevel = 1.
312 fluxLevel = 500
313 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []}
314 # Use different seeds for the science and template so every source is a diaSource
315 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
316 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
318 difference = science.clone()
319 difference.maskedImage -= matchedTemplate.maskedImage
320 detectionTask = self._setup_detection()
322 # Verify that detection runs without errors
323 detectionTask.run(science, matchedTemplate, difference)
325 def test_detect_dipoles(self):
326 """Run detection on a difference image containing dipoles.
327 """
328 # Set up the simulated images
329 noiseLevel = 1.
330 staticSeed = 1
331 fluxLevel = 1000
332 fluxRange = 1.5
333 nSources = 10
334 offset = 1
335 xSize = 300
336 ySize = 300
337 kernelSize = 32
338 # Avoid placing sources near the edge for this test, so that we can
339 # easily check that the correct number of sources are detected.
340 templateBorderSize = kernelSize//2
341 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
342 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
343 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
344 "xSize": xSize, "ySize": ySize}
345 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
346 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
347 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
348 difference = science.clone()
349 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
350 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
351 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
352 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
354 # Configure the detection Task
355 detectionTask = self._setup_detection(doMerge=False)
357 # Run detection and check the results
358 output = detectionTask.run(science, matchedTemplate, difference)
359 self.assertIn(dipoleFlag, output.diaSources.schema.getNames())
360 nSourcesDet = len(sources)
361 self.assertEqual(len(output.diaSources), 2*nSourcesDet)
362 refIds = []
363 # The diaSource check should fail if we don't merge positive and negative footprints
364 for diaSource in output.diaSources:
365 with self.assertRaises(AssertionError):
366 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
367 atol=np.sqrt(fluxRange*fluxLevel))
369 detectionTask2 = self._setup_detection(doMerge=True)
370 output2 = detectionTask2.run(science, matchedTemplate, difference)
371 self.assertEqual(len(output2.diaSources), nSourcesDet)
372 refIds = []
373 for diaSource in output2.diaSources:
374 if diaSource[dipoleFlag]:
375 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
376 rtol=0.05, atol=None, usePsfFlux=False)
377 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
378 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
379 else:
380 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
382 def test_sky_sources(self):
383 """Add sky sources and check that they are sufficiently far from other
384 sources and have negligible flux.
385 """
386 # Set up the simulated images
387 noiseLevel = 1.
388 staticSeed = 1
389 transientSeed = 6
390 transientFluxLevel = 1000.
391 transientFluxRange = 1.5
392 fluxLevel = 500
393 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
394 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
395 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
396 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
397 nSrc=10, fluxLevel=transientFluxLevel,
398 fluxRange=transientFluxRange,
399 noiseLevel=noiseLevel, noiseSeed=8)
400 difference = science.clone()
401 difference.maskedImage -= matchedTemplate.maskedImage
402 difference.maskedImage += transients.maskedImage
403 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2
405 # Configure the detection Task
406 detectionTask = self._setup_detection(doSkySources=True)
408 # Run detection and check the results
409 output = detectionTask.run(science, matchedTemplate, difference,
410 idFactory=self.idGenerator.make_table_id_factory())
411 skySources = output.diaSources[output.diaSources["sky_source"]]
412 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources)
413 for skySource in skySources:
414 # The sky sources should not be close to any other source
415 with self.assertRaises(AssertionError):
416 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
417 with self.assertRaises(AssertionError):
418 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
419 # The sky sources should have low flux levels.
420 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
421 atol=np.sqrt(transientFluxRange*transientFluxLevel))
423 # Catalog ids should be very large from this id generator.
424 self.assertTrue(all(output.diaSources['id'] > 1000000000))
426 def test_edge_detections(self):
427 """Sources with certain bad mask planes set should not be detected.
428 """
429 # Set up the simulated images
430 noiseLevel = 1.
431 staticSeed = 1
432 transientSeed = 6
433 fluxLevel = 500
434 radius = 2
435 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
436 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
437 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
439 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask
440 # Configure the detection Task
441 detectionTask = self._setup_detection()
442 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
443 nBad = len(excludeMaskPlanes)
444 self.assertGreater(nBad, 0)
445 kwargs["seed"] = transientSeed
446 kwargs["nSrc"] = nBad
447 kwargs["fluxLevel"] = 1000
449 # Run detection and check the results
450 def _detection_wrapper(setFlags=True):
451 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
452 difference = science.clone()
453 difference.maskedImage -= matchedTemplate.maskedImage
454 difference.maskedImage += transients.maskedImage
455 if setFlags:
456 for src, badMask in zip(transientSources, excludeMaskPlanes):
457 srcX = int(src.getX())
458 srcY = int(src.getY())
459 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
460 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
461 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
462 output = detectionTask.run(science, matchedTemplate, difference)
463 refIds = []
464 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes)
465 if setFlags:
466 self.assertEqual(np.sum(~goodSrcFlags), nBad)
467 else:
468 self.assertEqual(np.sum(~goodSrcFlags), 0)
469 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
470 if ~goodSrcFlag:
471 with self.assertRaises(AssertionError):
472 self._check_diaSource(transientSources, diaSource, refIds=refIds)
473 else:
474 self._check_diaSource(transientSources, diaSource, refIds=refIds)
475 _detection_wrapper(setFlags=False)
476 _detection_wrapper(setFlags=True)
478 def test_fake_mask_plane_propagation(self):
479 """Test that we have the mask planes related to fakes in diffim images.
480 This is testing method called updateMasks
481 """
482 xSize = 256
483 ySize = 256
484 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
485 science_fake_img, science_fake_sources = makeTestImage(
486 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1
487 )
488 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True)
489 tmplt_fake_img, tmplt_fake_sources = makeTestImage(
490 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1
491 )
492 # created fakes and added them to the images
493 science.image += science_fake_img.image
494 template.image += tmplt_fake_img.image
496 # TODO: DM-40796 update to INJECTED names when source injection gets refactored
497 # adding mask planes to both science and template images
498 science.mask.addMaskPlane("FAKE")
499 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE")
500 template.mask.addMaskPlane("FAKE")
501 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE")
503 # makeTestImage sets the DETECTED plane on the sources; we can use
504 # that to set the FAKE plane on the science and template images.
505 detected = science_fake_img.mask.getPlaneBitMask("DETECTED")
506 fake_pixels = (science_fake_img.mask.array & detected).nonzero()
507 science.mask.array[fake_pixels] |= science_fake_bitmask
508 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED")
509 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero()
510 template.mask.array[fake_pixels] |= science_fake_bitmask
512 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0
513 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0
515 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass()
516 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig)
517 subtraction = subtractTask.run(template, science, sources)
519 # check subtraction mask plane is set where we set the previous masks
520 diff_mask = subtraction.difference.mask
522 # science mask should be now in INJECTED
523 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0
525 # template mask should be now in INJECTED_TEMPLATE
526 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0
528 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int))
529 # The template is convolved, so the INJECTED_TEMPLATE mask plane may
530 # include more pixels than the FAKE mask plane
531 injTmplt_masked &= template_fake_masked
532 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int))
534 # Now check that detection of fakes have the correct flag for injections
535 detectionTask = self._setup_detection()
536 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
537 nBad = len(excludeMaskPlanes)
538 self.assertEqual(nBad, 1)
540 output = detectionTask.run(subtraction.matchedScience,
541 subtraction.matchedTemplate,
542 subtraction.difference)
544 sci_refIds = []
545 tmpl_refIds = []
546 for diaSrc in output.diaSources:
547 if diaSrc['base_PsfFlux_instFlux'] > 0:
548 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds)
549 self.assertTrue(diaSrc['base_PixelFlags_flag_injected'])
550 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter'])
551 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template'])
552 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
553 else:
554 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds)
555 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template'])
556 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter'])
557 self.assertFalse(diaSrc['base_PixelFlags_flag_injected'])
558 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter'])
561class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase):
562 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask
564 def test_detection_xy0(self):
565 """Basic functionality test with non-zero x0 and y0.
566 """
567 # Set up the simulated images
568 noiseLevel = 1.
569 staticSeed = 1
570 fluxLevel = 500
571 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890}
572 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
573 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
574 difference = science.clone()
575 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
576 scienceKernel = science.psf.getKernel()
577 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
579 # Configure the detection Task
580 detectionTask = self._setup_detection()
582 # Run detection and check the results
583 output = detectionTask.run(science, matchedTemplate, difference, score,
584 idFactory=self.idGenerator.make_table_id_factory())
586 # Catalog ids should be very large from this id generator.
587 self.assertTrue(all(output.diaSources['id'] > 1000000000))
588 subtractedMeasuredExposure = output.subtractedMeasuredExposure
590 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image)
592 def test_measurements_finite(self):
593 """Measured fluxes and centroids should always be finite.
594 """
595 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"]
597 # Set up the simulated images
598 noiseLevel = 1.
599 staticSeed = 1
600 transientSeed = 6
601 xSize = 256
602 ySize = 256
603 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0,
604 "xSize": xSize, "ySize": ySize}
605 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6,
606 nSrc=1, **kwargs)
607 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7,
608 nSrc=1, **kwargs)
609 rng = np.random.RandomState(3)
610 xLoc = np.arange(-5, xSize+5, 10)
611 rng.shuffle(xLoc)
612 yLoc = np.arange(-5, ySize+5, 10)
613 rng.shuffle(yLoc)
614 transients, transientSources = makeTestImage(seed=transientSeed,
615 nSrc=len(xLoc), fluxLevel=1000.,
616 noiseLevel=noiseLevel, noiseSeed=8,
617 xLoc=xLoc, yLoc=yLoc,
618 **kwargs)
619 difference = science.clone()
620 difference.maskedImage -= matchedTemplate.maskedImage
621 difference.maskedImage += transients.maskedImage
622 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
623 scienceKernel = science.psf.getKernel()
624 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
626 # Configure the detection Task
627 detectionTask = self._setup_detection(doForcedMeasurement=True)
629 # Run detection and check the results
630 output = detectionTask.run(science, matchedTemplate, difference, score)
632 for column in columnNames:
633 self._check_values(output.diaSources[column])
634 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize)
635 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize)
636 self._check_values(output.diaSources.getPsfInstFlux())
638 def test_detect_transients(self):
639 """Run detection on a difference image containing transients.
640 """
641 # Set up the simulated images
642 noiseLevel = 1.
643 staticSeed = 1
644 transientSeed = 6
645 fluxLevel = 500
646 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
647 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
648 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
649 scienceKernel = science.psf.getKernel()
650 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
652 # Configure the detection Task
653 detectionTask = self._setup_detection(doMerge=False)
654 kwargs["seed"] = transientSeed
655 kwargs["nSrc"] = 10
656 kwargs["fluxLevel"] = 1000
658 # Run detection and check the results
659 def _detection_wrapper(positive=True):
660 """Simulate positive or negative transients and run detection.
662 Parameters
663 ----------
664 positive : `bool`, optional
665 If set, use positive transient sources.
666 """
668 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
669 difference = science.clone()
670 difference.maskedImage -= matchedTemplate.maskedImage
671 if positive:
672 difference.maskedImage += transients.maskedImage
673 else:
674 difference.maskedImage -= transients.maskedImage
675 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
676 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the
677 # science image if we've e.g. removed parents post-deblending.
678 # Pass a clone of the science image, so that it doesn't disrupt
679 # later tests.
680 output = detectionTask.run(science.clone(), matchedTemplate, difference, score)
681 refIds = []
682 scale = 1. if positive else -1.
683 # sources near the edge may have untrustworthy centroids
684 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge']
685 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
686 if goodSrcFlag:
687 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale)
688 _detection_wrapper(positive=True)
689 _detection_wrapper(positive=False)
691 def test_detect_dipoles(self):
692 """Run detection on a difference image containing dipoles.
693 """
694 # Set up the simulated images
695 noiseLevel = 1.
696 staticSeed = 1
697 fluxLevel = 1000
698 fluxRange = 1.5
699 nSources = 10
700 offset = 1
701 xSize = 300
702 ySize = 300
703 kernelSize = 32
704 # Avoid placing sources near the edge for this test, so that we can
705 # easily check that the correct number of sources are detected.
706 templateBorderSize = kernelSize//2
707 dipoleFlag = "ip_diffim_DipoleFit_flag_classification"
708 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange,
709 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize,
710 "xSize": xSize, "ySize": ySize}
711 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
712 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
713 difference = science.clone()
714 # Shift the template by a pixel in order to make dipoles in the difference image.
715 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0)
716 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0)
717 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0)
718 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()]
719 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
720 scienceKernel = science.psf.getKernel()
721 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
723 # Configure the detection Task
724 detectionTask = self._setup_detection(doMerge=False)
726 # Run detection and check the results
727 output = detectionTask.run(science, matchedTemplate, difference, score)
728 self.assertIn(dipoleFlag, output.diaSources.schema.getNames())
729 nSourcesDet = len(sources)
730 # Since we did not merge the dipoles, each source should result in
731 # both a positive and a negative diaSource
732 self.assertEqual(len(output.diaSources), 2*nSourcesDet)
733 refIds = []
734 # The diaSource check should fail if we don't merge positive and negative footprints
735 for diaSource in output.diaSources:
736 with self.assertRaises(AssertionError):
737 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
738 atol=np.sqrt(fluxRange*fluxLevel))
740 detectionTask2 = self._setup_detection(doMerge=True)
741 output2 = detectionTask2.run(science, matchedTemplate, difference, score)
742 self.assertEqual(len(output2.diaSources), nSourcesDet)
743 refIds = []
744 for diaSource in output2.diaSources:
745 if diaSource[dipoleFlag]:
746 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0,
747 rtol=0.05, atol=None, usePsfFlux=False)
748 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.)
749 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1)
750 else:
751 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId())
753 def test_sky_sources(self):
754 """Add sky sources and check that they are sufficiently far from other
755 sources and have negligible flux.
756 """
757 # Set up the simulated images
758 noiseLevel = 1.
759 staticSeed = 1
760 transientSeed = 6
761 transientFluxLevel = 1000.
762 transientFluxRange = 1.5
763 fluxLevel = 500
764 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
765 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
766 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
767 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4,
768 nSrc=10, fluxLevel=transientFluxLevel,
769 fluxRange=transientFluxRange,
770 noiseLevel=noiseLevel, noiseSeed=8)
771 difference = science.clone()
772 difference.maskedImage -= matchedTemplate.maskedImage
773 difference.maskedImage += transients.maskedImage
774 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
775 scienceKernel = science.psf.getKernel()
776 kernelWidth = np.max(scienceKernel.getDimensions())//2
777 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
779 # Configure the detection Task
780 detectionTask = self._setup_detection(doSkySources=True)
782 # Run detection and check the results
783 output = detectionTask.run(science, matchedTemplate, difference, score,
784 idFactory=self.idGenerator.make_table_id_factory())
785 nSkySourcesGenerated = detectionTask.metadata["nSkySources"]
786 skySources = output.diaSources[output.diaSources["sky_source"]]
787 self.assertEqual(len(skySources), nSkySourcesGenerated)
788 for skySource in skySources:
789 # The sky sources should not be close to any other source
790 with self.assertRaises(AssertionError):
791 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth)
792 with self.assertRaises(AssertionError):
793 self._check_diaSource(sources, skySource, matchDistance=kernelWidth)
794 # The sky sources should have low flux levels.
795 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0.,
796 atol=np.sqrt(transientFluxRange*transientFluxLevel))
798 # Catalog ids should be very large from this id generator.
799 self.assertTrue(all(output.diaSources['id'] > 1000000000))
801 def test_edge_detections(self):
802 """Sources with certain bad mask planes set should not be detected.
803 """
804 # Set up the simulated images
805 noiseLevel = 1.
806 staticSeed = 1
807 transientSeed = 6
808 fluxLevel = 500
809 radius = 2
810 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel}
811 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs)
812 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs)
814 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask()
815 scienceKernel = science.psf.getKernel()
816 # Configure the detection Task
817 detectionTask = self._setup_detection()
818 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes
819 nBad = len(excludeMaskPlanes)
820 self.assertGreater(nBad, 0)
821 kwargs["seed"] = transientSeed
822 kwargs["nSrc"] = nBad
823 kwargs["fluxLevel"] = 1000
825 # Run detection and check the results
826 def _detection_wrapper(setFlags=True):
827 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs)
828 difference = science.clone()
829 difference.maskedImage -= matchedTemplate.maskedImage
830 difference.maskedImage += transients.maskedImage
831 if setFlags:
832 for src, badMask in zip(transientSources, excludeMaskPlanes):
833 srcX = int(src.getX())
834 srcY = int(src.getY())
835 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius),
836 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1))
837 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask)
838 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl)
839 output = detectionTask.run(science, matchedTemplate, difference, score)
840 refIds = []
841 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes)
842 if setFlags:
843 self.assertEqual(np.sum(~goodSrcFlags), nBad)
844 else:
845 self.assertEqual(np.sum(~goodSrcFlags), 0)
846 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags):
847 if ~goodSrcFlag:
848 with self.assertRaises(AssertionError):
849 self._check_diaSource(transientSources, diaSource, refIds=refIds)
850 else:
851 self._check_diaSource(transientSources, diaSource, refIds=refIds)
852 _detection_wrapper(setFlags=False)
853 _detection_wrapper(setFlags=True)
856def setup_module(module):
857 lsst.utils.tests.init()
860class MemoryTestCase(lsst.utils.tests.MemoryTestCase):
861 pass
864if __name__ == "__main__": 864 ↛ 865line 864 didn't jump to line 865, because the condition on line 864 was never true
865 lsst.utils.tests.init()
866 unittest.main()