Coverage for tests/test_detectAndMeasure.py: 6%

509 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-23 04:19 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import unittest 

24 

25import lsst.geom 

26from lsst.ip.diffim import detectAndMeasure, subtractImages 

27from lsst.ip.diffim.utils import makeTestImage 

28from lsst.pipe.base import InvalidQuantumError 

29import lsst.utils.tests 

30 

31 

32class DetectAndMeasureTestBase: 

33 

34 def _check_diaSource(self, refSources, diaSource, refIds=None, 

35 matchDistance=1., scale=1., usePsfFlux=True, 

36 rtol=0.021, atol=None): 

37 """Match a diaSource with a source in a reference catalog 

38 and compare properties. 

39 

40 Parameters 

41 ---------- 

42 refSources : `lsst.afw.table.SourceCatalog` 

43 The reference catalog. 

44 diaSource : `lsst.afw.table.SourceRecord` 

45 The new diaSource to match to the reference catalog. 

46 refIds : `list` of `int`, optional 

47 Source IDs of previously associated diaSources. 

48 matchDistance : `float`, optional 

49 Maximum distance allowed between the detected and reference source 

50 locations, in pixels. 

51 scale : `float`, optional 

52 Optional factor to scale the flux by before performing the test. 

53 usePsfFlux : `bool`, optional 

54 If set, test the PsfInstFlux field, otherwise use ApInstFlux. 

55 rtol : `float`, optional 

56 Relative tolerance of the flux value test. 

57 atol : `float`, optional 

58 Absolute tolerance of the flux value test. 

59 """ 

60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2 

61 + (diaSource.getY() - refSources.getY())**2) 

62 self.assertLess(min(distance), matchDistance) 

63 src = refSources[np.argmin(distance)] 

64 if refIds is not None: 

65 # Check that the same source was not previously associated 

66 self.assertNotIn(src.getId(), refIds) 

67 refIds.append(src.getId()) 

68 if atol is None: 

69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux() 

70 if usePsfFlux: 

71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(), 

72 rtol=rtol, atol=atol) 

73 else: 

74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(), 

75 rtol=rtol, atol=atol) 

76 

77 def _check_values(self, values, minValue=None, maxValue=None): 

78 """Verify that an array has finite values, and optionally that they are 

79 within specified minimum and maximum bounds. 

80 

81 Parameters 

82 ---------- 

83 values : `numpy.ndarray` 

84 Array of values to check. 

85 minValue : `float`, optional 

86 Minimum allowable value. 

87 maxValue : `float`, optional 

88 Maximum allowable value. 

89 """ 

90 self.assertTrue(np.all(np.isfinite(values))) 

91 if minValue is not None: 

92 self.assertTrue(np.all(values >= minValue)) 

93 if maxValue is not None: 

94 self.assertTrue(np.all(values <= maxValue)) 

95 

96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs): 

97 """Setup and configure the detection and measurement PipelineTask. 

98 

99 Parameters 

100 ---------- 

101 doSkySources : `bool`, optional 

102 Generate sky sources. 

103 nSkySources : `int`, optional 

104 The number of sky sources to add in isolated background regions. 

105 **kwargs 

106 Any additional config parameters to set. 

107 

108 Returns 

109 ------- 

110 `lsst.pipe.base.PipelineTask` 

111 The configured Task to use for detection and measurement. 

112 """ 

113 config = self.detectionTask.ConfigClass() 

114 config.doSkySources = doSkySources 

115 if doSkySources: 

116 config.skySources.nSources = nSkySources 

117 config.update(**kwargs) 

118 

119 # Make a realistic id generator so that output catalog ids are useful. 

120 dataId = lsst.daf.butler.DataCoordinate.standardize( 

121 instrument="I", 

122 visit=42, 

123 detector=12, 

124 universe=lsst.daf.butler.DimensionUniverse(), 

125 ) 

126 config.idGenerator.packer.name = "observation" 

127 config.idGenerator.packer["observation"].n_observations = 10000 

128 config.idGenerator.packer["observation"].n_detectors = 99 

129 config.idGenerator.n_releases = 8 

130 config.idGenerator.release_id = 2 

131 self.idGenerator = config.idGenerator.apply(dataId) 

132 

133 return self.detectionTask(config=config) 

134 

135 

136class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

137 detectionTask = detectAndMeasure.DetectAndMeasureTask 

138 

139 def test_detection_xy0(self): 

140 """Basic functionality test with non-zero x0 and y0. 

141 """ 

142 # Set up the simulated images 

143 noiseLevel = 1. 

144 staticSeed = 1 

145 fluxLevel = 500 

146 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

147 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

148 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

149 difference = science.clone() 

150 

151 # Configure the detection Task 

152 detectionTask = self._setup_detection() 

153 

154 # Run detection and check the results 

155 output = detectionTask.run(science, matchedTemplate, difference, 

156 idFactory=self.idGenerator.make_table_id_factory()) 

157 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

158 

159 # Catalog ids should be very large from this id generator. 

160 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

161 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

162 

163 def test_measurements_finite(self): 

164 """Measured fluxes and centroids should always be finite. 

165 """ 

166 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

167 

168 # Set up the simulated images 

169 noiseLevel = 1. 

170 staticSeed = 1 

171 transientSeed = 6 

172 xSize = 256 

173 ySize = 256 

174 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

175 "xSize": xSize, "ySize": ySize} 

176 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

177 nSrc=1, **kwargs) 

178 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

179 nSrc=1, **kwargs) 

180 rng = np.random.RandomState(3) 

181 xLoc = np.arange(-5, xSize+5, 10) 

182 rng.shuffle(xLoc) 

183 yLoc = np.arange(-5, ySize+5, 10) 

184 rng.shuffle(yLoc) 

185 transients, transientSources = makeTestImage(seed=transientSeed, 

186 nSrc=len(xLoc), fluxLevel=1000., 

187 noiseLevel=noiseLevel, noiseSeed=8, 

188 xLoc=xLoc, yLoc=yLoc, 

189 **kwargs) 

190 difference = science.clone() 

191 difference.maskedImage -= matchedTemplate.maskedImage 

192 difference.maskedImage += transients.maskedImage 

193 

194 # Configure the detection Task 

195 detectionTask = self._setup_detection(doForcedMeasurement=True) 

196 

197 # Run detection and check the results 

198 output = detectionTask.run(science, matchedTemplate, difference) 

199 

200 for column in columnNames: 

201 self._check_values(output.diaSources[column]) 

202 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

203 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

204 self._check_values(output.diaSources.getPsfInstFlux()) 

205 

206 def test_raise_config_schema_mismatch(self): 

207 """Check that sources with specified flags are removed from the catalog. 

208 """ 

209 # Configure the detection Task, and and set a config that is not in the schema 

210 with self.assertRaises(InvalidQuantumError): 

211 self._setup_detection(badSourceFlags=["Bogus_flag_42"]) 

212 

213 def test_remove_unphysical(self): 

214 """Check that sources with specified flags are removed from the catalog. 

215 """ 

216 # Set up the simulated images 

217 noiseLevel = 1. 

218 staticSeed = 1 

219 xSize = 256 

220 ySize = 256 

221 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} 

222 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

223 nSrc=1, **kwargs) 

224 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

225 nSrc=1, **kwargs) 

226 difference = science.clone() 

227 bbox = difference.getBBox() 

228 difference.maskedImage -= matchedTemplate.maskedImage 

229 

230 # Configure the detection Task, and do not remove unphysical sources 

231 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

232 badSourceFlags=[]) 

233 

234 # Run detection and check the results 

235 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

236 badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

237 nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove) 

238 # Verify that unphysical sources exist 

239 self.assertGreater(nBadNoRemove, 0) 

240 

241 # Configure the detection Task, and remove unphysical sources 

242 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

243 badSourceFlags=["base_PixelFlags_flag_offimage", ]) 

244 

245 # Run detection and check the results 

246 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

247 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

248 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) 

249 # Verify that all sources are physical 

250 self.assertEqual(nBadDoRemove, 0) 

251 # Set a few centroids outside the image bounding box 

252 nSetBad = 5 

253 for src in diaSources[0: nSetBad]: 

254 src["slot_Centroid_x"] += xSize 

255 src["slot_Centroid_y"] += ySize 

256 src["base_PixelFlags_flag_offimage"] = True 

257 # Verify that these sources are outside the image 

258 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

259 nBad = np.count_nonzero(badDiaSrc) 

260 self.assertEqual(nBad, nSetBad) 

261 diaSourcesNoBad = detectionTask._removeBadSources(diaSources) 

262 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) 

263 

264 # Verify that no sources outside the image bounding box remain 

265 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) 

266 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) 

267 

268 def test_detect_transients(self): 

269 """Run detection on a difference image containing transients. 

270 """ 

271 # Set up the simulated images 

272 noiseLevel = 1. 

273 staticSeed = 1 

274 transientSeed = 6 

275 fluxLevel = 500 

276 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

277 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

278 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

279 

280 # Configure the detection Task 

281 detectionTask = self._setup_detection(doMerge=False) 

282 kwargs["seed"] = transientSeed 

283 kwargs["nSrc"] = 10 

284 kwargs["fluxLevel"] = 1000 

285 

286 # Run detection and check the results 

287 def _detection_wrapper(positive=True): 

288 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

289 difference = science.clone() 

290 difference.maskedImage -= matchedTemplate.maskedImage 

291 if positive: 

292 difference.maskedImage += transients.maskedImage 

293 else: 

294 difference.maskedImage -= transients.maskedImage 

295 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

296 # science image if we've e.g. removed parents post-deblending. 

297 # Pass a clone of the science image, so that it doesn't disrupt 

298 # later tests. 

299 output = detectionTask.run(science.clone(), matchedTemplate, difference) 

300 refIds = [] 

301 scale = 1. if positive else -1. 

302 for diaSource in output.diaSources: 

303 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

304 _detection_wrapper(positive=True) 

305 _detection_wrapper(positive=False) 

306 

307 def test_missing_mask_planes(self): 

308 """Check that detection runs with missing mask planes. 

309 """ 

310 # Set up the simulated images 

311 noiseLevel = 1. 

312 fluxLevel = 500 

313 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []} 

314 # Use different seeds for the science and template so every source is a diaSource 

315 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

316 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

317 

318 difference = science.clone() 

319 difference.maskedImage -= matchedTemplate.maskedImage 

320 detectionTask = self._setup_detection() 

321 

322 # Verify that detection runs without errors 

323 detectionTask.run(science, matchedTemplate, difference) 

324 

325 def test_detect_dipoles(self): 

326 """Run detection on a difference image containing dipoles. 

327 """ 

328 # Set up the simulated images 

329 noiseLevel = 1. 

330 staticSeed = 1 

331 fluxLevel = 1000 

332 fluxRange = 1.5 

333 nSources = 10 

334 offset = 1 

335 xSize = 300 

336 ySize = 300 

337 kernelSize = 32 

338 # Avoid placing sources near the edge for this test, so that we can 

339 # easily check that the correct number of sources are detected. 

340 templateBorderSize = kernelSize//2 

341 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

342 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

343 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

344 "xSize": xSize, "ySize": ySize} 

345 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

346 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

347 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

348 difference = science.clone() 

349 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

350 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

351 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

352 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

353 

354 # Configure the detection Task 

355 detectionTask = self._setup_detection(doMerge=False) 

356 

357 # Run detection and check the results 

358 output = detectionTask.run(science, matchedTemplate, difference) 

359 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

360 nSourcesDet = len(sources) 

361 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

362 refIds = [] 

363 # The diaSource check should fail if we don't merge positive and negative footprints 

364 for diaSource in output.diaSources: 

365 with self.assertRaises(AssertionError): 

366 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

367 atol=np.sqrt(fluxRange*fluxLevel)) 

368 

369 detectionTask2 = self._setup_detection(doMerge=True) 

370 output2 = detectionTask2.run(science, matchedTemplate, difference) 

371 self.assertEqual(len(output2.diaSources), nSourcesDet) 

372 refIds = [] 

373 for diaSource in output2.diaSources: 

374 if diaSource[dipoleFlag]: 

375 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

376 rtol=0.05, atol=None, usePsfFlux=False) 

377 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

378 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

379 else: 

380 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

381 

382 def test_sky_sources(self): 

383 """Add sky sources and check that they are sufficiently far from other 

384 sources and have negligible flux. 

385 """ 

386 # Set up the simulated images 

387 noiseLevel = 1. 

388 staticSeed = 1 

389 transientSeed = 6 

390 transientFluxLevel = 1000. 

391 transientFluxRange = 1.5 

392 fluxLevel = 500 

393 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

394 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

395 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

396 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

397 nSrc=10, fluxLevel=transientFluxLevel, 

398 fluxRange=transientFluxRange, 

399 noiseLevel=noiseLevel, noiseSeed=8) 

400 difference = science.clone() 

401 difference.maskedImage -= matchedTemplate.maskedImage 

402 difference.maskedImage += transients.maskedImage 

403 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2 

404 

405 # Configure the detection Task 

406 detectionTask = self._setup_detection(doSkySources=True) 

407 

408 # Run detection and check the results 

409 output = detectionTask.run(science, matchedTemplate, difference, 

410 idFactory=self.idGenerator.make_table_id_factory()) 

411 skySources = output.diaSources[output.diaSources["sky_source"]] 

412 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources) 

413 for skySource in skySources: 

414 # The sky sources should not be close to any other source 

415 with self.assertRaises(AssertionError): 

416 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

417 with self.assertRaises(AssertionError): 

418 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

419 # The sky sources should have low flux levels. 

420 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

421 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

422 

423 # Catalog ids should be very large from this id generator. 

424 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

425 

426 def test_edge_detections(self): 

427 """Sources with certain bad mask planes set should not be detected. 

428 """ 

429 # Set up the simulated images 

430 noiseLevel = 1. 

431 staticSeed = 1 

432 transientSeed = 6 

433 fluxLevel = 500 

434 radius = 2 

435 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

436 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

437 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

438 

439 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask 

440 # Configure the detection Task 

441 detectionTask = self._setup_detection() 

442 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

443 nBad = len(excludeMaskPlanes) 

444 self.assertGreater(nBad, 0) 

445 kwargs["seed"] = transientSeed 

446 kwargs["nSrc"] = nBad 

447 kwargs["fluxLevel"] = 1000 

448 

449 # Run detection and check the results 

450 def _detection_wrapper(setFlags=True): 

451 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

452 difference = science.clone() 

453 difference.maskedImage -= matchedTemplate.maskedImage 

454 difference.maskedImage += transients.maskedImage 

455 if setFlags: 

456 for src, badMask in zip(transientSources, excludeMaskPlanes): 

457 srcX = int(src.getX()) 

458 srcY = int(src.getY()) 

459 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

460 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

461 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

462 output = detectionTask.run(science, matchedTemplate, difference) 

463 refIds = [] 

464 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes) 

465 if setFlags: 

466 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

467 else: 

468 self.assertEqual(np.sum(~goodSrcFlags), 0) 

469 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

470 if ~goodSrcFlag: 

471 with self.assertRaises(AssertionError): 

472 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

473 else: 

474 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

475 _detection_wrapper(setFlags=False) 

476 _detection_wrapper(setFlags=True) 

477 

478 def test_fake_mask_plane_propagation(self): 

479 """Test that we have the mask planes related to fakes in diffim images. 

480 This is testing method called updateMasks 

481 """ 

482 xSize = 256 

483 ySize = 256 

484 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

485 science_fake_img, science_fake_sources = makeTestImage( 

486 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1 

487 ) 

488 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

489 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

490 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1 

491 ) 

492 # created fakes and added them to the images 

493 science.image += science_fake_img.image 

494 template.image += tmplt_fake_img.image 

495 

496 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

497 # adding mask planes to both science and template images 

498 science.mask.addMaskPlane("FAKE") 

499 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE") 

500 template.mask.addMaskPlane("FAKE") 

501 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE") 

502 

503 # makeTestImage sets the DETECTED plane on the sources; we can use 

504 # that to set the FAKE plane on the science and template images. 

505 detected = science_fake_img.mask.getPlaneBitMask("DETECTED") 

506 fake_pixels = (science_fake_img.mask.array & detected).nonzero() 

507 science.mask.array[fake_pixels] |= science_fake_bitmask 

508 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED") 

509 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero() 

510 template.mask.array[fake_pixels] |= science_fake_bitmask 

511 

512 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0 

513 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 

514 

515 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

516 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) 

517 subtraction = subtractTask.run(template, science, sources) 

518 

519 # check subtraction mask plane is set where we set the previous masks 

520 diff_mask = subtraction.difference.mask 

521 

522 # science mask should be now in INJECTED 

523 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

524 

525 # template mask should be now in INJECTED_TEMPLATE 

526 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

527 

528 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int)) 

529 # The template is convolved, so the INJECTED_TEMPLATE mask plane may 

530 # include more pixels than the FAKE mask plane 

531 injTmplt_masked &= template_fake_masked 

532 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int)) 

533 

534 # Now check that detection of fakes have the correct flag for injections 

535 detectionTask = self._setup_detection() 

536 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

537 nBad = len(excludeMaskPlanes) 

538 self.assertEqual(nBad, 1) 

539 

540 output = detectionTask.run(subtraction.matchedScience, 

541 subtraction.matchedTemplate, 

542 subtraction.difference) 

543 

544 sci_refIds = [] 

545 tmpl_refIds = [] 

546 for diaSrc in output.diaSources: 

547 if diaSrc['base_PsfFlux_instFlux'] > 0: 

548 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds) 

549 self.assertTrue(diaSrc['base_PixelFlags_flag_injected']) 

550 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter']) 

551 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template']) 

552 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

553 else: 

554 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds) 

555 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template']) 

556 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

557 self.assertFalse(diaSrc['base_PixelFlags_flag_injected']) 

558 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter']) 

559 

560 

561class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

562 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask 

563 

564 def test_detection_xy0(self): 

565 """Basic functionality test with non-zero x0 and y0. 

566 """ 

567 # Set up the simulated images 

568 noiseLevel = 1. 

569 staticSeed = 1 

570 fluxLevel = 500 

571 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

572 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

573 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

574 difference = science.clone() 

575 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

576 scienceKernel = science.psf.getKernel() 

577 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

578 

579 # Configure the detection Task 

580 detectionTask = self._setup_detection() 

581 

582 # Run detection and check the results 

583 output = detectionTask.run(science, matchedTemplate, difference, score, 

584 idFactory=self.idGenerator.make_table_id_factory()) 

585 

586 # Catalog ids should be very large from this id generator. 

587 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

588 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

589 

590 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

591 

592 def test_measurements_finite(self): 

593 """Measured fluxes and centroids should always be finite. 

594 """ 

595 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

596 

597 # Set up the simulated images 

598 noiseLevel = 1. 

599 staticSeed = 1 

600 transientSeed = 6 

601 xSize = 256 

602 ySize = 256 

603 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

604 "xSize": xSize, "ySize": ySize} 

605 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

606 nSrc=1, **kwargs) 

607 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

608 nSrc=1, **kwargs) 

609 rng = np.random.RandomState(3) 

610 xLoc = np.arange(-5, xSize+5, 10) 

611 rng.shuffle(xLoc) 

612 yLoc = np.arange(-5, ySize+5, 10) 

613 rng.shuffle(yLoc) 

614 transients, transientSources = makeTestImage(seed=transientSeed, 

615 nSrc=len(xLoc), fluxLevel=1000., 

616 noiseLevel=noiseLevel, noiseSeed=8, 

617 xLoc=xLoc, yLoc=yLoc, 

618 **kwargs) 

619 difference = science.clone() 

620 difference.maskedImage -= matchedTemplate.maskedImage 

621 difference.maskedImage += transients.maskedImage 

622 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

623 scienceKernel = science.psf.getKernel() 

624 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

625 

626 # Configure the detection Task 

627 detectionTask = self._setup_detection(doForcedMeasurement=True) 

628 

629 # Run detection and check the results 

630 output = detectionTask.run(science, matchedTemplate, difference, score) 

631 

632 for column in columnNames: 

633 self._check_values(output.diaSources[column]) 

634 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

635 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

636 self._check_values(output.diaSources.getPsfInstFlux()) 

637 

638 def test_detect_transients(self): 

639 """Run detection on a difference image containing transients. 

640 """ 

641 # Set up the simulated images 

642 noiseLevel = 1. 

643 staticSeed = 1 

644 transientSeed = 6 

645 fluxLevel = 500 

646 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

647 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

648 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

649 scienceKernel = science.psf.getKernel() 

650 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

651 

652 # Configure the detection Task 

653 detectionTask = self._setup_detection(doMerge=False) 

654 kwargs["seed"] = transientSeed 

655 kwargs["nSrc"] = 10 

656 kwargs["fluxLevel"] = 1000 

657 

658 # Run detection and check the results 

659 def _detection_wrapper(positive=True): 

660 """Simulate positive or negative transients and run detection. 

661 

662 Parameters 

663 ---------- 

664 positive : `bool`, optional 

665 If set, use positive transient sources. 

666 """ 

667 

668 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

669 difference = science.clone() 

670 difference.maskedImage -= matchedTemplate.maskedImage 

671 if positive: 

672 difference.maskedImage += transients.maskedImage 

673 else: 

674 difference.maskedImage -= transients.maskedImage 

675 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

676 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

677 # science image if we've e.g. removed parents post-deblending. 

678 # Pass a clone of the science image, so that it doesn't disrupt 

679 # later tests. 

680 output = detectionTask.run(science.clone(), matchedTemplate, difference, score) 

681 refIds = [] 

682 scale = 1. if positive else -1. 

683 # sources near the edge may have untrustworthy centroids 

684 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge'] 

685 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

686 if goodSrcFlag: 

687 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

688 _detection_wrapper(positive=True) 

689 _detection_wrapper(positive=False) 

690 

691 def test_detect_dipoles(self): 

692 """Run detection on a difference image containing dipoles. 

693 """ 

694 # Set up the simulated images 

695 noiseLevel = 1. 

696 staticSeed = 1 

697 fluxLevel = 1000 

698 fluxRange = 1.5 

699 nSources = 10 

700 offset = 1 

701 xSize = 300 

702 ySize = 300 

703 kernelSize = 32 

704 # Avoid placing sources near the edge for this test, so that we can 

705 # easily check that the correct number of sources are detected. 

706 templateBorderSize = kernelSize//2 

707 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

708 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

709 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

710 "xSize": xSize, "ySize": ySize} 

711 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

712 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

713 difference = science.clone() 

714 # Shift the template by a pixel in order to make dipoles in the difference image. 

715 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

716 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

717 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

718 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

719 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

720 scienceKernel = science.psf.getKernel() 

721 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

722 

723 # Configure the detection Task 

724 detectionTask = self._setup_detection(doMerge=False) 

725 

726 # Run detection and check the results 

727 output = detectionTask.run(science, matchedTemplate, difference, score) 

728 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

729 nSourcesDet = len(sources) 

730 # Since we did not merge the dipoles, each source should result in 

731 # both a positive and a negative diaSource 

732 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

733 refIds = [] 

734 # The diaSource check should fail if we don't merge positive and negative footprints 

735 for diaSource in output.diaSources: 

736 with self.assertRaises(AssertionError): 

737 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

738 atol=np.sqrt(fluxRange*fluxLevel)) 

739 

740 detectionTask2 = self._setup_detection(doMerge=True) 

741 output2 = detectionTask2.run(science, matchedTemplate, difference, score) 

742 self.assertEqual(len(output2.diaSources), nSourcesDet) 

743 refIds = [] 

744 for diaSource in output2.diaSources: 

745 if diaSource[dipoleFlag]: 

746 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

747 rtol=0.05, atol=None, usePsfFlux=False) 

748 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

749 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

750 else: 

751 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

752 

753 def test_sky_sources(self): 

754 """Add sky sources and check that they are sufficiently far from other 

755 sources and have negligible flux. 

756 """ 

757 # Set up the simulated images 

758 noiseLevel = 1. 

759 staticSeed = 1 

760 transientSeed = 6 

761 transientFluxLevel = 1000. 

762 transientFluxRange = 1.5 

763 fluxLevel = 500 

764 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

765 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

766 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

767 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

768 nSrc=10, fluxLevel=transientFluxLevel, 

769 fluxRange=transientFluxRange, 

770 noiseLevel=noiseLevel, noiseSeed=8) 

771 difference = science.clone() 

772 difference.maskedImage -= matchedTemplate.maskedImage 

773 difference.maskedImage += transients.maskedImage 

774 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

775 scienceKernel = science.psf.getKernel() 

776 kernelWidth = np.max(scienceKernel.getDimensions())//2 

777 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

778 

779 # Configure the detection Task 

780 detectionTask = self._setup_detection(doSkySources=True) 

781 

782 # Run detection and check the results 

783 output = detectionTask.run(science, matchedTemplate, difference, score, 

784 idFactory=self.idGenerator.make_table_id_factory()) 

785 nSkySourcesGenerated = detectionTask.metadata["nSkySources"] 

786 skySources = output.diaSources[output.diaSources["sky_source"]] 

787 self.assertEqual(len(skySources), nSkySourcesGenerated) 

788 for skySource in skySources: 

789 # The sky sources should not be close to any other source 

790 with self.assertRaises(AssertionError): 

791 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

792 with self.assertRaises(AssertionError): 

793 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

794 # The sky sources should have low flux levels. 

795 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

796 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

797 

798 # Catalog ids should be very large from this id generator. 

799 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

800 

801 def test_edge_detections(self): 

802 """Sources with certain bad mask planes set should not be detected. 

803 """ 

804 # Set up the simulated images 

805 noiseLevel = 1. 

806 staticSeed = 1 

807 transientSeed = 6 

808 fluxLevel = 500 

809 radius = 2 

810 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

811 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

812 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

813 

814 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

815 scienceKernel = science.psf.getKernel() 

816 # Configure the detection Task 

817 detectionTask = self._setup_detection() 

818 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

819 nBad = len(excludeMaskPlanes) 

820 self.assertGreater(nBad, 0) 

821 kwargs["seed"] = transientSeed 

822 kwargs["nSrc"] = nBad 

823 kwargs["fluxLevel"] = 1000 

824 

825 # Run detection and check the results 

826 def _detection_wrapper(setFlags=True): 

827 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

828 difference = science.clone() 

829 difference.maskedImage -= matchedTemplate.maskedImage 

830 difference.maskedImage += transients.maskedImage 

831 if setFlags: 

832 for src, badMask in zip(transientSources, excludeMaskPlanes): 

833 srcX = int(src.getX()) 

834 srcY = int(src.getY()) 

835 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

836 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

837 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

838 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

839 output = detectionTask.run(science, matchedTemplate, difference, score) 

840 refIds = [] 

841 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes) 

842 if setFlags: 

843 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

844 else: 

845 self.assertEqual(np.sum(~goodSrcFlags), 0) 

846 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

847 if ~goodSrcFlag: 

848 with self.assertRaises(AssertionError): 

849 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

850 else: 

851 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

852 _detection_wrapper(setFlags=False) 

853 _detection_wrapper(setFlags=True) 

854 

855 

856def setup_module(module): 

857 lsst.utils.tests.init() 

858 

859 

860class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

861 pass 

862 

863 

864if __name__ == "__main__": 864 ↛ 865line 864 didn't jump to line 865, because the condition on line 864 was never true

865 lsst.utils.tests.init() 

866 unittest.main()