Coverage for tests/test_detectAndMeasure.py: 6%

507 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-05-04 03:09 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import unittest 

24 

25import lsst.geom 

26from lsst.ip.diffim import detectAndMeasure, subtractImages 

27from lsst.ip.diffim.utils import makeTestImage 

28from lsst.pipe.base import InvalidQuantumError 

29import lsst.utils.tests 

30 

31 

32class DetectAndMeasureTestBase: 

33 

34 def _check_diaSource(self, refSources, diaSource, refIds=None, 

35 matchDistance=1., scale=1., usePsfFlux=True, 

36 rtol=0.025, atol=None): 

37 """Match a diaSource with a source in a reference catalog 

38 and compare properties. 

39 

40 Parameters 

41 ---------- 

42 refSources : `lsst.afw.table.SourceCatalog` 

43 The reference catalog. 

44 diaSource : `lsst.afw.table.SourceRecord` 

45 The new diaSource to match to the reference catalog. 

46 refIds : `list` of `int`, optional 

47 Source IDs of previously associated diaSources. 

48 matchDistance : `float`, optional 

49 Maximum distance allowed between the detected and reference source 

50 locations, in pixels. 

51 scale : `float`, optional 

52 Optional factor to scale the flux by before performing the test. 

53 usePsfFlux : `bool`, optional 

54 If set, test the PsfInstFlux field, otherwise use ApInstFlux. 

55 rtol : `float`, optional 

56 Relative tolerance of the flux value test. 

57 atol : `float`, optional 

58 Absolute tolerance of the flux value test. 

59 """ 

60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2 

61 + (diaSource.getY() - refSources.getY())**2) 

62 self.assertLess(min(distance), matchDistance) 

63 src = refSources[np.argmin(distance)] 

64 if refIds is not None: 

65 # Check that the same source was not previously associated 

66 self.assertNotIn(src.getId(), refIds) 

67 refIds.append(src.getId()) 

68 if atol is None: 

69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux() 

70 if usePsfFlux: 

71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(), 

72 rtol=rtol, atol=atol) 

73 else: 

74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(), 

75 rtol=rtol, atol=atol) 

76 

77 def _check_values(self, values, minValue=None, maxValue=None): 

78 """Verify that an array has finite values, and optionally that they are 

79 within specified minimum and maximum bounds. 

80 

81 Parameters 

82 ---------- 

83 values : `numpy.ndarray` 

84 Array of values to check. 

85 minValue : `float`, optional 

86 Minimum allowable value. 

87 maxValue : `float`, optional 

88 Maximum allowable value. 

89 """ 

90 self.assertTrue(np.all(np.isfinite(values))) 

91 if minValue is not None: 

92 self.assertTrue(np.all(values >= minValue)) 

93 if maxValue is not None: 

94 self.assertTrue(np.all(values <= maxValue)) 

95 

96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs): 

97 """Setup and configure the detection and measurement PipelineTask. 

98 

99 Parameters 

100 ---------- 

101 doSkySources : `bool`, optional 

102 Generate sky sources. 

103 nSkySources : `int`, optional 

104 The number of sky sources to add in isolated background regions. 

105 **kwargs 

106 Any additional config parameters to set. 

107 

108 Returns 

109 ------- 

110 `lsst.pipe.base.PipelineTask` 

111 The configured Task to use for detection and measurement. 

112 """ 

113 config = self.detectionTask.ConfigClass() 

114 config.doSkySources = doSkySources 

115 if doSkySources: 

116 config.skySources.nSources = nSkySources 

117 config.update(**kwargs) 

118 

119 # Make a realistic id generator so that output catalog ids are useful. 

120 dataId = lsst.daf.butler.DataCoordinate.standardize( 

121 instrument="I", 

122 visit=42, 

123 detector=12, 

124 universe=lsst.daf.butler.DimensionUniverse(), 

125 ) 

126 config.idGenerator.packer.name = "observation" 

127 config.idGenerator.packer["observation"].n_observations = 10000 

128 config.idGenerator.packer["observation"].n_detectors = 99 

129 config.idGenerator.n_releases = 8 

130 config.idGenerator.release_id = 2 

131 self.idGenerator = config.idGenerator.apply(dataId) 

132 

133 return self.detectionTask(config=config) 

134 

135 

136class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

137 detectionTask = detectAndMeasure.DetectAndMeasureTask 

138 

139 def test_detection_xy0(self): 

140 """Basic functionality test with non-zero x0 and y0. 

141 """ 

142 # Set up the simulated images 

143 noiseLevel = 1. 

144 staticSeed = 1 

145 fluxLevel = 500 

146 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

147 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

148 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

149 difference = science.clone() 

150 

151 # Configure the detection Task 

152 detectionTask = self._setup_detection() 

153 

154 # Run detection and check the results 

155 output = detectionTask.run(science, matchedTemplate, difference, 

156 idFactory=self.idGenerator.make_table_id_factory()) 

157 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

158 

159 # Catalog ids should be very large from this id generator. 

160 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

161 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

162 

163 def test_measurements_finite(self): 

164 """Measured fluxes and centroids should always be finite. 

165 """ 

166 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

167 

168 # Set up the simulated images 

169 noiseLevel = 1. 

170 staticSeed = 1 

171 transientSeed = 6 

172 xSize = 256 

173 ySize = 256 

174 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

175 "xSize": xSize, "ySize": ySize} 

176 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

177 nSrc=1, **kwargs) 

178 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

179 nSrc=1, **kwargs) 

180 rng = np.random.RandomState(3) 

181 xLoc = np.arange(-5, xSize+5, 10) 

182 rng.shuffle(xLoc) 

183 yLoc = np.arange(-5, ySize+5, 10) 

184 rng.shuffle(yLoc) 

185 transients, transientSources = makeTestImage(seed=transientSeed, 

186 nSrc=len(xLoc), fluxLevel=1000., 

187 noiseLevel=noiseLevel, noiseSeed=8, 

188 xLoc=xLoc, yLoc=yLoc, 

189 **kwargs) 

190 difference = science.clone() 

191 difference.maskedImage -= matchedTemplate.maskedImage 

192 difference.maskedImage += transients.maskedImage 

193 

194 # Configure the detection Task 

195 detectionTask = self._setup_detection(doForcedMeasurement=True) 

196 

197 # Run detection and check the results 

198 output = detectionTask.run(science, matchedTemplate, difference) 

199 

200 for column in columnNames: 

201 self._check_values(output.diaSources[column]) 

202 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

203 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

204 self._check_values(output.diaSources.getPsfInstFlux()) 

205 

206 def test_raise_config_schema_mismatch(self): 

207 """Check that sources with specified flags are removed from the catalog. 

208 """ 

209 # Configure the detection Task, and and set a config that is not in the schema 

210 with self.assertRaises(InvalidQuantumError): 

211 self._setup_detection(badSourceFlags=["Bogus_flag_42"]) 

212 

213 def test_remove_unphysical(self): 

214 """Check that sources with specified flags are removed from the catalog. 

215 """ 

216 # Set up the simulated images 

217 noiseLevel = 1. 

218 staticSeed = 1 

219 xSize = 256 

220 ySize = 256 

221 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} 

222 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

223 nSrc=1, **kwargs) 

224 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

225 nSrc=1, **kwargs) 

226 difference = science.clone() 

227 bbox = difference.getBBox() 

228 difference.maskedImage -= matchedTemplate.maskedImage 

229 

230 # Configure the detection Task, and remove unphysical sources 

231 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

232 badSourceFlags=["base_PixelFlags_flag_offimage", ]) 

233 

234 # Run detection and check the results 

235 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

236 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

237 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) 

238 # Verify that all sources are physical 

239 self.assertEqual(nBadDoRemove, 0) 

240 # Set a few centroids outside the image bounding box 

241 nSetBad = 5 

242 for src in diaSources[0: nSetBad]: 

243 src["slot_Centroid_x"] += xSize 

244 src["slot_Centroid_y"] += ySize 

245 src["base_PixelFlags_flag_offimage"] = True 

246 # Verify that these sources are outside the image 

247 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

248 nBad = np.count_nonzero(badDiaSrc) 

249 self.assertEqual(nBad, nSetBad) 

250 diaSourcesNoBad = detectionTask._removeBadSources(diaSources) 

251 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) 

252 

253 # Verify that no sources outside the image bounding box remain 

254 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) 

255 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) 

256 

257 def test_detect_transients(self): 

258 """Run detection on a difference image containing transients. 

259 """ 

260 # Set up the simulated images 

261 noiseLevel = 1. 

262 staticSeed = 1 

263 transientSeed = 6 

264 fluxLevel = 500 

265 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

266 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

267 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

268 

269 # Configure the detection Task 

270 detectionTask = self._setup_detection(doMerge=False) 

271 kwargs["seed"] = transientSeed 

272 kwargs["nSrc"] = 10 

273 kwargs["fluxLevel"] = 1000 

274 

275 # Run detection and check the results 

276 def _detection_wrapper(positive=True): 

277 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

278 difference = science.clone() 

279 difference.maskedImage -= matchedTemplate.maskedImage 

280 if positive: 

281 difference.maskedImage += transients.maskedImage 

282 else: 

283 difference.maskedImage -= transients.maskedImage 

284 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

285 # science image if we've e.g. removed parents post-deblending. 

286 # Pass a clone of the science image, so that it doesn't disrupt 

287 # later tests. 

288 output = detectionTask.run(science.clone(), matchedTemplate, difference) 

289 refIds = [] 

290 scale = 1. if positive else -1. 

291 for diaSource in output.diaSources: 

292 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

293 _detection_wrapper(positive=True) 

294 _detection_wrapper(positive=False) 

295 

296 def test_missing_mask_planes(self): 

297 """Check that detection runs with missing mask planes. 

298 """ 

299 # Set up the simulated images 

300 noiseLevel = 1. 

301 fluxLevel = 500 

302 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []} 

303 # Use different seeds for the science and template so every source is a diaSource 

304 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

305 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

306 

307 difference = science.clone() 

308 difference.maskedImage -= matchedTemplate.maskedImage 

309 detectionTask = self._setup_detection() 

310 

311 # Verify that detection runs without errors 

312 detectionTask.run(science, matchedTemplate, difference) 

313 

314 def test_detect_dipoles(self): 

315 """Run detection on a difference image containing dipoles. 

316 """ 

317 # Set up the simulated images 

318 noiseLevel = 1. 

319 staticSeed = 1 

320 fluxLevel = 1000 

321 fluxRange = 1.5 

322 nSources = 10 

323 offset = 1 

324 xSize = 300 

325 ySize = 300 

326 kernelSize = 32 

327 # Avoid placing sources near the edge for this test, so that we can 

328 # easily check that the correct number of sources are detected. 

329 templateBorderSize = kernelSize//2 

330 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

331 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

332 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

333 "xSize": xSize, "ySize": ySize} 

334 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

335 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

336 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

337 difference = science.clone() 

338 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

339 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

340 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

341 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

342 

343 detectionTask = self._setup_detection(doMerge=True) 

344 output = detectionTask.run(science, matchedTemplate, difference) 

345 self.assertEqual(len(output.diaSources), len(sources)) 

346 refIds = [] 

347 for diaSource in output.diaSources: 

348 if diaSource[dipoleFlag]: 

349 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

350 rtol=0.05, atol=None, usePsfFlux=False) 

351 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

352 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

353 else: 

354 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

355 

356 def test_sky_sources(self): 

357 """Add sky sources and check that they are sufficiently far from other 

358 sources and have negligible flux. 

359 """ 

360 # Set up the simulated images 

361 noiseLevel = 1. 

362 staticSeed = 1 

363 transientSeed = 6 

364 transientFluxLevel = 1000. 

365 transientFluxRange = 1.5 

366 fluxLevel = 500 

367 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

368 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

369 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

370 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

371 nSrc=10, fluxLevel=transientFluxLevel, 

372 fluxRange=transientFluxRange, 

373 noiseLevel=noiseLevel, noiseSeed=8) 

374 difference = science.clone() 

375 difference.maskedImage -= matchedTemplate.maskedImage 

376 difference.maskedImage += transients.maskedImage 

377 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2 

378 

379 # Configure the detection Task 

380 detectionTask = self._setup_detection(doSkySources=True) 

381 

382 # Run detection and check the results 

383 output = detectionTask.run(science, matchedTemplate, difference, 

384 idFactory=self.idGenerator.make_table_id_factory()) 

385 skySources = output.diaSources[output.diaSources["sky_source"]] 

386 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources) 

387 for skySource in skySources: 

388 # The sky sources should not be close to any other source 

389 with self.assertRaises(AssertionError): 

390 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

391 with self.assertRaises(AssertionError): 

392 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

393 # The sky sources should have low flux levels. 

394 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

395 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

396 

397 # Catalog ids should be very large from this id generator. 

398 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

399 

400 def test_edge_detections(self): 

401 """Sources with certain bad mask planes set should not be detected. 

402 """ 

403 # Set up the simulated images 

404 noiseLevel = 1. 

405 staticSeed = 1 

406 transientSeed = 6 

407 fluxLevel = 500 

408 radius = 2 

409 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

410 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

411 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

412 

413 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask 

414 # Configure the detection Task 

415 detectionTask = self._setup_detection() 

416 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

417 nBad = len(excludeMaskPlanes) 

418 self.assertGreater(nBad, 0) 

419 kwargs["seed"] = transientSeed 

420 kwargs["nSrc"] = nBad 

421 kwargs["fluxLevel"] = 1000 

422 

423 # Run detection and check the results 

424 def _detection_wrapper(setFlags=True): 

425 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

426 difference = science.clone() 

427 difference.maskedImage -= matchedTemplate.maskedImage 

428 difference.maskedImage += transients.maskedImage 

429 if setFlags: 

430 for src, badMask in zip(transientSources, excludeMaskPlanes): 

431 srcX = int(src.getX()) 

432 srcY = int(src.getY()) 

433 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

434 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

435 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

436 output = detectionTask.run(science, matchedTemplate, difference) 

437 refIds = [] 

438 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes) 

439 if setFlags: 

440 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

441 else: 

442 self.assertEqual(np.sum(~goodSrcFlags), 0) 

443 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

444 if ~goodSrcFlag: 

445 with self.assertRaises(AssertionError): 

446 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

447 else: 

448 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

449 _detection_wrapper(setFlags=False) 

450 _detection_wrapper(setFlags=True) 

451 

452 def test_fake_mask_plane_propagation(self): 

453 """Test that we have the mask planes related to fakes in diffim images. 

454 This is testing method called updateMasks 

455 """ 

456 xSize = 256 

457 ySize = 256 

458 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

459 science_fake_img, science_fake_sources = makeTestImage( 

460 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1 

461 ) 

462 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

463 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

464 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1 

465 ) 

466 # created fakes and added them to the images 

467 science.image += science_fake_img.image 

468 template.image += tmplt_fake_img.image 

469 

470 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

471 # adding mask planes to both science and template images 

472 science.mask.addMaskPlane("FAKE") 

473 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE") 

474 template.mask.addMaskPlane("FAKE") 

475 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE") 

476 

477 # makeTestImage sets the DETECTED plane on the sources; we can use 

478 # that to set the FAKE plane on the science and template images. 

479 detected = science_fake_img.mask.getPlaneBitMask("DETECTED") 

480 fake_pixels = (science_fake_img.mask.array & detected).nonzero() 

481 science.mask.array[fake_pixels] |= science_fake_bitmask 

482 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED") 

483 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero() 

484 template.mask.array[fake_pixels] |= science_fake_bitmask 

485 

486 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0 

487 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 

488 

489 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

490 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) 

491 subtraction = subtractTask.run(template, science, sources) 

492 

493 # check subtraction mask plane is set where we set the previous masks 

494 diff_mask = subtraction.difference.mask 

495 

496 # science mask should be now in INJECTED 

497 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

498 

499 # template mask should be now in INJECTED_TEMPLATE 

500 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

501 

502 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int)) 

503 # The template is convolved, so the INJECTED_TEMPLATE mask plane may 

504 # include more pixels than the FAKE mask plane 

505 injTmplt_masked &= template_fake_masked 

506 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int)) 

507 

508 # Now check that detection of fakes have the correct flag for injections 

509 detectionTask = self._setup_detection() 

510 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

511 nBad = len(excludeMaskPlanes) 

512 self.assertEqual(nBad, 1) 

513 

514 output = detectionTask.run(subtraction.matchedScience, 

515 subtraction.matchedTemplate, 

516 subtraction.difference) 

517 

518 sci_refIds = [] 

519 tmpl_refIds = [] 

520 for diaSrc in output.diaSources: 

521 if diaSrc['base_PsfFlux_instFlux'] > 0: 

522 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds) 

523 self.assertTrue(diaSrc['base_PixelFlags_flag_injected']) 

524 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter']) 

525 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template']) 

526 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

527 else: 

528 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds) 

529 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template']) 

530 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

531 self.assertFalse(diaSrc['base_PixelFlags_flag_injected']) 

532 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter']) 

533 

534 def test_mask_streaks(self): 

535 """Run detection on a difference image containing a streak. 

536 """ 

537 # Set up the simulated images 

538 noiseLevel = 1. 

539 staticSeed = 1 

540 fluxLevel = 500 

541 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

542 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

543 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

544 

545 # Configure the detection Task 

546 detectionTask = self._setup_detection(doMerge=False, doMaskStreaks=True) 

547 

548 # Test that no streaks are detected 

549 difference = science.clone() 

550 difference.maskedImage -= matchedTemplate.maskedImage 

551 output = detectionTask.run(science, matchedTemplate, difference) 

552 outMask = output.subtractedMeasuredExposure.mask.array 

553 streakMask = output.subtractedMeasuredExposure.mask.getPlaneBitMask("STREAK") 

554 streakMaskSet = (outMask & streakMask) > 0 

555 self.assertTrue(np.all(streakMaskSet == 0)) 

556 

557 # Add streak-like shape and check that streak is detected 

558 difference.image.array[20:23, 40:200] += 50 

559 output = detectionTask.run(science, matchedTemplate, difference) 

560 outMask = output.subtractedMeasuredExposure.mask.array 

561 streakMask = output.subtractedMeasuredExposure.mask.getPlaneBitMask("STREAK") 

562 streakMaskSet = (outMask & streakMask) > 0 

563 self.assertTrue(np.all(streakMaskSet[20:23, 40:200])) 

564 

565 

566class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

567 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask 

568 

569 def test_detection_xy0(self): 

570 """Basic functionality test with non-zero x0 and y0. 

571 """ 

572 # Set up the simulated images 

573 noiseLevel = 1. 

574 staticSeed = 1 

575 fluxLevel = 500 

576 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

577 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

578 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

579 difference = science.clone() 

580 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

581 scienceKernel = science.psf.getKernel() 

582 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

583 

584 # Configure the detection Task 

585 detectionTask = self._setup_detection() 

586 

587 # Run detection and check the results 

588 output = detectionTask.run(science, matchedTemplate, difference, score, 

589 idFactory=self.idGenerator.make_table_id_factory()) 

590 

591 # Catalog ids should be very large from this id generator. 

592 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

593 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

594 

595 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

596 

597 def test_measurements_finite(self): 

598 """Measured fluxes and centroids should always be finite. 

599 """ 

600 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

601 

602 # Set up the simulated images 

603 noiseLevel = 1. 

604 staticSeed = 1 

605 transientSeed = 6 

606 xSize = 256 

607 ySize = 256 

608 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

609 "xSize": xSize, "ySize": ySize} 

610 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

611 nSrc=1, **kwargs) 

612 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

613 nSrc=1, **kwargs) 

614 rng = np.random.RandomState(3) 

615 xLoc = np.arange(-5, xSize+5, 10) 

616 rng.shuffle(xLoc) 

617 yLoc = np.arange(-5, ySize+5, 10) 

618 rng.shuffle(yLoc) 

619 transients, transientSources = makeTestImage(seed=transientSeed, 

620 nSrc=len(xLoc), fluxLevel=1000., 

621 noiseLevel=noiseLevel, noiseSeed=8, 

622 xLoc=xLoc, yLoc=yLoc, 

623 **kwargs) 

624 difference = science.clone() 

625 difference.maskedImage -= matchedTemplate.maskedImage 

626 difference.maskedImage += transients.maskedImage 

627 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

628 scienceKernel = science.psf.getKernel() 

629 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

630 

631 # Configure the detection Task 

632 detectionTask = self._setup_detection(doForcedMeasurement=True) 

633 

634 # Run detection and check the results 

635 output = detectionTask.run(science, matchedTemplate, difference, score) 

636 

637 for column in columnNames: 

638 self._check_values(output.diaSources[column]) 

639 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

640 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

641 self._check_values(output.diaSources.getPsfInstFlux()) 

642 

643 def test_detect_transients(self): 

644 """Run detection on a difference image containing transients. 

645 """ 

646 # Set up the simulated images 

647 noiseLevel = 1. 

648 staticSeed = 1 

649 transientSeed = 6 

650 fluxLevel = 500 

651 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

652 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

653 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

654 scienceKernel = science.psf.getKernel() 

655 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

656 

657 # Configure the detection Task 

658 detectionTask = self._setup_detection(doMerge=False) 

659 kwargs["seed"] = transientSeed 

660 kwargs["nSrc"] = 10 

661 kwargs["fluxLevel"] = 1000 

662 

663 # Run detection and check the results 

664 def _detection_wrapper(positive=True): 

665 """Simulate positive or negative transients and run detection. 

666 

667 Parameters 

668 ---------- 

669 positive : `bool`, optional 

670 If set, use positive transient sources. 

671 """ 

672 

673 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

674 difference = science.clone() 

675 difference.maskedImage -= matchedTemplate.maskedImage 

676 if positive: 

677 difference.maskedImage += transients.maskedImage 

678 else: 

679 difference.maskedImage -= transients.maskedImage 

680 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

681 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

682 # science image if we've e.g. removed parents post-deblending. 

683 # Pass a clone of the science image, so that it doesn't disrupt 

684 # later tests. 

685 output = detectionTask.run(science.clone(), matchedTemplate, difference, score) 

686 refIds = [] 

687 scale = 1. if positive else -1. 

688 # sources near the edge may have untrustworthy centroids 

689 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge'] 

690 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

691 if goodSrcFlag: 

692 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

693 _detection_wrapper(positive=True) 

694 _detection_wrapper(positive=False) 

695 

696 def test_detect_dipoles(self): 

697 """Run detection on a difference image containing dipoles. 

698 """ 

699 # Set up the simulated images 

700 noiseLevel = 1. 

701 staticSeed = 1 

702 fluxLevel = 1000 

703 fluxRange = 1.5 

704 nSources = 10 

705 offset = 1 

706 xSize = 300 

707 ySize = 300 

708 kernelSize = 32 

709 # Avoid placing sources near the edge for this test, so that we can 

710 # easily check that the correct number of sources are detected. 

711 templateBorderSize = kernelSize//2 

712 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

713 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

714 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

715 "xSize": xSize, "ySize": ySize} 

716 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

717 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

718 difference = science.clone() 

719 # Shift the template by a pixel in order to make dipoles in the difference image. 

720 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

721 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

722 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

723 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

724 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

725 scienceKernel = science.psf.getKernel() 

726 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

727 

728 detectionTask = self._setup_detection() 

729 output = detectionTask.run(science, matchedTemplate, difference, score) 

730 self.assertEqual(len(output.diaSources), len(sources)) 

731 refIds = [] 

732 for diaSource in output.diaSources: 

733 if diaSource[dipoleFlag]: 

734 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

735 rtol=0.05, atol=None, usePsfFlux=False) 

736 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

737 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

738 else: 

739 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

740 

741 def test_sky_sources(self): 

742 """Add sky sources and check that they are sufficiently far from other 

743 sources and have negligible flux. 

744 """ 

745 # Set up the simulated images 

746 noiseLevel = 1. 

747 staticSeed = 1 

748 transientSeed = 6 

749 transientFluxLevel = 1000. 

750 transientFluxRange = 1.5 

751 fluxLevel = 500 

752 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

753 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

754 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

755 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

756 nSrc=10, fluxLevel=transientFluxLevel, 

757 fluxRange=transientFluxRange, 

758 noiseLevel=noiseLevel, noiseSeed=8) 

759 difference = science.clone() 

760 difference.maskedImage -= matchedTemplate.maskedImage 

761 difference.maskedImage += transients.maskedImage 

762 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

763 scienceKernel = science.psf.getKernel() 

764 kernelWidth = np.max(scienceKernel.getDimensions())//2 

765 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

766 

767 # Configure the detection Task 

768 detectionTask = self._setup_detection(doSkySources=True) 

769 

770 # Run detection and check the results 

771 output = detectionTask.run(science, matchedTemplate, difference, score, 

772 idFactory=self.idGenerator.make_table_id_factory()) 

773 nSkySourcesGenerated = detectionTask.metadata["n_skySources"] 

774 skySources = output.diaSources[output.diaSources["sky_source"]] 

775 self.assertEqual(len(skySources), nSkySourcesGenerated) 

776 for skySource in skySources: 

777 # The sky sources should not be close to any other source 

778 with self.assertRaises(AssertionError): 

779 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

780 with self.assertRaises(AssertionError): 

781 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

782 # The sky sources should have low flux levels. 

783 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

784 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

785 

786 # Catalog ids should be very large from this id generator. 

787 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

788 

789 def test_edge_detections(self): 

790 """Sources with certain bad mask planes set should not be detected. 

791 """ 

792 # Set up the simulated images 

793 noiseLevel = 1. 

794 staticSeed = 1 

795 transientSeed = 6 

796 fluxLevel = 500 

797 radius = 2 

798 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

799 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

800 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

801 

802 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

803 scienceKernel = science.psf.getKernel() 

804 # Configure the detection Task 

805 detectionTask = self._setup_detection() 

806 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

807 nBad = len(excludeMaskPlanes) 

808 self.assertGreater(nBad, 0) 

809 kwargs["seed"] = transientSeed 

810 kwargs["nSrc"] = nBad 

811 kwargs["fluxLevel"] = 1000 

812 

813 # Run detection and check the results 

814 def _detection_wrapper(setFlags=True): 

815 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

816 difference = science.clone() 

817 difference.maskedImage -= matchedTemplate.maskedImage 

818 difference.maskedImage += transients.maskedImage 

819 if setFlags: 

820 for src, badMask in zip(transientSources, excludeMaskPlanes): 

821 srcX = int(src.getX()) 

822 srcY = int(src.getY()) 

823 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

824 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

825 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

826 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

827 output = detectionTask.run(science, matchedTemplate, difference, score) 

828 refIds = [] 

829 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes) 

830 if setFlags: 

831 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

832 else: 

833 self.assertEqual(np.sum(~goodSrcFlags), 0) 

834 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

835 if ~goodSrcFlag: 

836 with self.assertRaises(AssertionError): 

837 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

838 else: 

839 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

840 _detection_wrapper(setFlags=False) 

841 _detection_wrapper(setFlags=True) 

842 

843 

844def setup_module(module): 

845 lsst.utils.tests.init() 

846 

847 

848class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

849 pass 

850 

851 

852if __name__ == "__main__": 852 ↛ 853line 852 didn't jump to line 853, because the condition on line 852 was never true

853 lsst.utils.tests.init() 

854 unittest.main()