Coverage for tests/test_detectAndMeasure.py: 6%

487 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-11 03:14 -0700

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import unittest 

24 

25import lsst.geom 

26from lsst.ip.diffim import detectAndMeasure, subtractImages 

27from lsst.ip.diffim.utils import makeTestImage 

28from lsst.pipe.base import InvalidQuantumError 

29import lsst.utils.tests 

30 

31 

32class DetectAndMeasureTestBase: 

33 

34 def _check_diaSource(self, refSources, diaSource, refIds=None, 

35 matchDistance=1., scale=1., usePsfFlux=True, 

36 rtol=0.025, atol=None): 

37 """Match a diaSource with a source in a reference catalog 

38 and compare properties. 

39 

40 Parameters 

41 ---------- 

42 refSources : `lsst.afw.table.SourceCatalog` 

43 The reference catalog. 

44 diaSource : `lsst.afw.table.SourceRecord` 

45 The new diaSource to match to the reference catalog. 

46 refIds : `list` of `int`, optional 

47 Source IDs of previously associated diaSources. 

48 matchDistance : `float`, optional 

49 Maximum distance allowed between the detected and reference source 

50 locations, in pixels. 

51 scale : `float`, optional 

52 Optional factor to scale the flux by before performing the test. 

53 usePsfFlux : `bool`, optional 

54 If set, test the PsfInstFlux field, otherwise use ApInstFlux. 

55 rtol : `float`, optional 

56 Relative tolerance of the flux value test. 

57 atol : `float`, optional 

58 Absolute tolerance of the flux value test. 

59 """ 

60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2 

61 + (diaSource.getY() - refSources.getY())**2) 

62 self.assertLess(min(distance), matchDistance) 

63 src = refSources[np.argmin(distance)] 

64 if refIds is not None: 

65 # Check that the same source was not previously associated 

66 self.assertNotIn(src.getId(), refIds) 

67 refIds.append(src.getId()) 

68 if atol is None: 

69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux() 

70 if usePsfFlux: 

71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(), 

72 rtol=rtol, atol=atol) 

73 else: 

74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(), 

75 rtol=rtol, atol=atol) 

76 

77 def _check_values(self, values, minValue=None, maxValue=None): 

78 """Verify that an array has finite values, and optionally that they are 

79 within specified minimum and maximum bounds. 

80 

81 Parameters 

82 ---------- 

83 values : `numpy.ndarray` 

84 Array of values to check. 

85 minValue : `float`, optional 

86 Minimum allowable value. 

87 maxValue : `float`, optional 

88 Maximum allowable value. 

89 """ 

90 self.assertTrue(np.all(np.isfinite(values))) 

91 if minValue is not None: 

92 self.assertTrue(np.all(values >= minValue)) 

93 if maxValue is not None: 

94 self.assertTrue(np.all(values <= maxValue)) 

95 

96 def _setup_detection(self, doSkySources=False, nSkySources=5, doWriteMetrics=False, **kwargs): 

97 """Setup and configure the detection and measurement PipelineTask. 

98 

99 Parameters 

100 ---------- 

101 doSkySources : `bool`, optional 

102 Generate sky sources. 

103 nSkySources : `int`, optional 

104 The number of sky sources to add in isolated background regions. 

105 **kwargs 

106 Any additional config parameters to set. 

107 

108 Returns 

109 ------- 

110 `lsst.pipe.base.PipelineTask` 

111 The configured Task to use for detection and measurement. 

112 """ 

113 config = self.detectionTask.ConfigClass() 

114 config.doSkySources = doSkySources 

115 if doSkySources: 

116 config.skySources.nSources = nSkySources 

117 config.doWriteMetrics = doWriteMetrics 

118 config.update(**kwargs) 

119 

120 # Make a realistic id generator so that output catalog ids are useful. 

121 dataId = lsst.daf.butler.DataCoordinate.standardize( 

122 instrument="I", 

123 visit=42, 

124 detector=12, 

125 universe=lsst.daf.butler.DimensionUniverse(), 

126 ) 

127 config.idGenerator.packer.name = "observation" 

128 config.idGenerator.packer["observation"].n_observations = 10000 

129 config.idGenerator.packer["observation"].n_detectors = 99 

130 config.idGenerator.n_releases = 8 

131 config.idGenerator.release_id = 2 

132 self.idGenerator = config.idGenerator.apply(dataId) 

133 

134 return self.detectionTask(config=config) 

135 

136 

137class DetectAndMeasureTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

138 detectionTask = detectAndMeasure.DetectAndMeasureTask 

139 

140 def test_detection_xy0(self): 

141 """Basic functionality test with non-zero x0 and y0. 

142 """ 

143 # Set up the simulated images 

144 noiseLevel = 1. 

145 staticSeed = 1 

146 fluxLevel = 500 

147 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

148 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

149 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

150 difference = science.clone() 

151 

152 # Configure the detection Task 

153 detectionTask = self._setup_detection() 

154 

155 # Run detection and check the results 

156 output = detectionTask.run(science, matchedTemplate, difference, 

157 idFactory=self.idGenerator.make_table_id_factory()) 

158 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

159 

160 # Catalog ids should be very large from this id generator. 

161 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

162 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

163 

164 def test_measurements_finite(self): 

165 """Measured fluxes and centroids should always be finite. 

166 """ 

167 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

168 

169 # Set up the simulated images 

170 noiseLevel = 1. 

171 staticSeed = 1 

172 transientSeed = 6 

173 xSize = 256 

174 ySize = 256 

175 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

176 "xSize": xSize, "ySize": ySize} 

177 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

178 nSrc=1, **kwargs) 

179 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

180 nSrc=1, **kwargs) 

181 rng = np.random.RandomState(3) 

182 xLoc = np.arange(-5, xSize+5, 10) 

183 rng.shuffle(xLoc) 

184 yLoc = np.arange(-5, ySize+5, 10) 

185 rng.shuffle(yLoc) 

186 transients, transientSources = makeTestImage(seed=transientSeed, 

187 nSrc=len(xLoc), fluxLevel=1000., 

188 noiseLevel=noiseLevel, noiseSeed=8, 

189 xLoc=xLoc, yLoc=yLoc, 

190 **kwargs) 

191 difference = science.clone() 

192 difference.maskedImage -= matchedTemplate.maskedImage 

193 difference.maskedImage += transients.maskedImage 

194 

195 # Configure the detection Task 

196 detectionTask = self._setup_detection(doForcedMeasurement=True) 

197 

198 # Run detection and check the results 

199 output = detectionTask.run(science, matchedTemplate, difference) 

200 

201 for column in columnNames: 

202 self._check_values(output.diaSources[column]) 

203 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

204 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

205 self._check_values(output.diaSources.getPsfInstFlux()) 

206 

207 def test_raise_config_schema_mismatch(self): 

208 """Check that sources with specified flags are removed from the catalog. 

209 """ 

210 # Configure the detection Task, and and set a config that is not in the schema 

211 with self.assertRaises(InvalidQuantumError): 

212 self._setup_detection(badSourceFlags=["Bogus_flag_42"]) 

213 

214 def test_remove_unphysical(self): 

215 """Check that sources with specified flags are removed from the catalog. 

216 """ 

217 # Set up the simulated images 

218 noiseLevel = 1. 

219 staticSeed = 1 

220 xSize = 256 

221 ySize = 256 

222 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} 

223 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

224 nSrc=1, **kwargs) 

225 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

226 nSrc=1, **kwargs) 

227 difference = science.clone() 

228 bbox = difference.getBBox() 

229 difference.maskedImage -= matchedTemplate.maskedImage 

230 

231 # Configure the detection Task, and remove unphysical sources 

232 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

233 badSourceFlags=["base_PixelFlags_flag_offimage", ]) 

234 

235 # Run detection and check the results 

236 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

237 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

238 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) 

239 # Verify that all sources are physical 

240 self.assertEqual(nBadDoRemove, 0) 

241 # Set a few centroids outside the image bounding box 

242 nSetBad = 5 

243 for src in diaSources[0: nSetBad]: 

244 src["slot_Centroid_x"] += xSize 

245 src["slot_Centroid_y"] += ySize 

246 src["base_PixelFlags_flag_offimage"] = True 

247 # Verify that these sources are outside the image 

248 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

249 nBad = np.count_nonzero(badDiaSrc) 

250 self.assertEqual(nBad, nSetBad) 

251 diaSourcesNoBad = detectionTask._removeBadSources(diaSources) 

252 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) 

253 

254 # Verify that no sources outside the image bounding box remain 

255 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) 

256 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) 

257 

258 def test_detect_transients(self): 

259 """Run detection on a difference image containing transients. 

260 """ 

261 # Set up the simulated images 

262 noiseLevel = 1. 

263 staticSeed = 1 

264 transientSeed = 6 

265 fluxLevel = 500 

266 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

267 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

268 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

269 

270 # Configure the detection Task 

271 detectionTask = self._setup_detection(doMerge=False) 

272 kwargs["seed"] = transientSeed 

273 kwargs["nSrc"] = 10 

274 kwargs["fluxLevel"] = 1000 

275 

276 # Run detection and check the results 

277 def _detection_wrapper(positive=True): 

278 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

279 difference = science.clone() 

280 difference.maskedImage -= matchedTemplate.maskedImage 

281 if positive: 

282 difference.maskedImage += transients.maskedImage 

283 else: 

284 difference.maskedImage -= transients.maskedImage 

285 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

286 # science image if we've e.g. removed parents post-deblending. 

287 # Pass a clone of the science image, so that it doesn't disrupt 

288 # later tests. 

289 output = detectionTask.run(science.clone(), matchedTemplate, difference) 

290 refIds = [] 

291 scale = 1. if positive else -1. 

292 for diaSource in output.diaSources: 

293 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

294 _detection_wrapper(positive=True) 

295 _detection_wrapper(positive=False) 

296 

297 def test_missing_mask_planes(self): 

298 """Check that detection runs with missing mask planes. 

299 """ 

300 # Set up the simulated images 

301 noiseLevel = 1. 

302 fluxLevel = 500 

303 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []} 

304 # Use different seeds for the science and template so every source is a diaSource 

305 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

306 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

307 

308 difference = science.clone() 

309 difference.maskedImage -= matchedTemplate.maskedImage 

310 detectionTask = self._setup_detection() 

311 

312 # Verify that detection runs without errors 

313 detectionTask.run(science, matchedTemplate, difference) 

314 

315 def test_detect_dipoles(self): 

316 """Run detection on a difference image containing dipoles. 

317 """ 

318 # Set up the simulated images 

319 noiseLevel = 1. 

320 staticSeed = 1 

321 fluxLevel = 1000 

322 fluxRange = 1.5 

323 nSources = 10 

324 offset = 1 

325 xSize = 300 

326 ySize = 300 

327 kernelSize = 32 

328 # Avoid placing sources near the edge for this test, so that we can 

329 # easily check that the correct number of sources are detected. 

330 templateBorderSize = kernelSize//2 

331 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

332 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

333 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

334 "xSize": xSize, "ySize": ySize} 

335 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

336 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

337 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

338 difference = science.clone() 

339 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

340 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

341 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

342 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

343 

344 detectionTask = self._setup_detection(doMerge=True) 

345 output = detectionTask.run(science, matchedTemplate, difference) 

346 self.assertEqual(len(output.diaSources), len(sources)) 

347 refIds = [] 

348 for diaSource in output.diaSources: 

349 if diaSource[dipoleFlag]: 

350 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

351 rtol=0.05, atol=None, usePsfFlux=False) 

352 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

353 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

354 else: 

355 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

356 

357 def test_sky_sources(self): 

358 """Add sky sources and check that they are sufficiently far from other 

359 sources and have negligible flux. 

360 """ 

361 # Set up the simulated images 

362 noiseLevel = 1. 

363 staticSeed = 1 

364 transientSeed = 6 

365 transientFluxLevel = 1000. 

366 transientFluxRange = 1.5 

367 fluxLevel = 500 

368 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

369 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

370 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

371 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

372 nSrc=10, fluxLevel=transientFluxLevel, 

373 fluxRange=transientFluxRange, 

374 noiseLevel=noiseLevel, noiseSeed=8) 

375 difference = science.clone() 

376 difference.maskedImage -= matchedTemplate.maskedImage 

377 difference.maskedImage += transients.maskedImage 

378 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2 

379 

380 # Configure the detection Task 

381 detectionTask = self._setup_detection(doSkySources=True) 

382 

383 # Run detection and check the results 

384 output = detectionTask.run(science, matchedTemplate, difference, 

385 idFactory=self.idGenerator.make_table_id_factory()) 

386 skySources = output.diaSources[output.diaSources["sky_source"]] 

387 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources) 

388 for skySource in skySources: 

389 # The sky sources should not be close to any other source 

390 with self.assertRaises(AssertionError): 

391 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

392 with self.assertRaises(AssertionError): 

393 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

394 # The sky sources should have low flux levels. 

395 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

396 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

397 

398 # Catalog ids should be very large from this id generator. 

399 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

400 

401 def test_edge_detections(self): 

402 """Sources with certain bad mask planes set should not be detected. 

403 """ 

404 # Set up the simulated images 

405 noiseLevel = 1. 

406 staticSeed = 1 

407 transientSeed = 6 

408 fluxLevel = 500 

409 radius = 2 

410 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

411 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

412 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

413 

414 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask 

415 # Configure the detection Task 

416 detectionTask = self._setup_detection() 

417 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

418 nBad = len(excludeMaskPlanes) 

419 self.assertGreater(nBad, 0) 

420 kwargs["seed"] = transientSeed 

421 kwargs["nSrc"] = nBad 

422 kwargs["fluxLevel"] = 1000 

423 

424 # Run detection and check the results 

425 def _detection_wrapper(setFlags=True): 

426 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

427 difference = science.clone() 

428 difference.maskedImage -= matchedTemplate.maskedImage 

429 difference.maskedImage += transients.maskedImage 

430 if setFlags: 

431 for src, badMask in zip(transientSources, excludeMaskPlanes): 

432 srcX = int(src.getX()) 

433 srcY = int(src.getY()) 

434 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

435 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

436 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

437 output = detectionTask.run(science, matchedTemplate, difference) 

438 refIds = [] 

439 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes) 

440 if setFlags: 

441 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

442 else: 

443 self.assertEqual(np.sum(~goodSrcFlags), 0) 

444 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

445 if ~goodSrcFlag: 

446 with self.assertRaises(AssertionError): 

447 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

448 else: 

449 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

450 _detection_wrapper(setFlags=False) 

451 _detection_wrapper(setFlags=True) 

452 

453 def test_fake_mask_plane_propagation(self): 

454 """Test that we have the mask planes related to fakes in diffim images. 

455 This is testing method called updateMasks 

456 """ 

457 xSize = 256 

458 ySize = 256 

459 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

460 science_fake_img, science_fake_sources = makeTestImage( 

461 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1 

462 ) 

463 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

464 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

465 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1 

466 ) 

467 # created fakes and added them to the images 

468 science.image += science_fake_img.image 

469 template.image += tmplt_fake_img.image 

470 

471 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

472 # adding mask planes to both science and template images 

473 science.mask.addMaskPlane("FAKE") 

474 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE") 

475 template.mask.addMaskPlane("FAKE") 

476 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE") 

477 

478 # makeTestImage sets the DETECTED plane on the sources; we can use 

479 # that to set the FAKE plane on the science and template images. 

480 detected = science_fake_img.mask.getPlaneBitMask("DETECTED") 

481 fake_pixels = (science_fake_img.mask.array & detected).nonzero() 

482 science.mask.array[fake_pixels] |= science_fake_bitmask 

483 detected = tmplt_fake_img.mask.getPlaneBitMask("DETECTED") 

484 fake_pixels = (tmplt_fake_img.mask.array & detected).nonzero() 

485 template.mask.array[fake_pixels] |= science_fake_bitmask 

486 

487 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0 

488 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 

489 

490 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

491 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) 

492 subtraction = subtractTask.run(template, science, sources) 

493 

494 # check subtraction mask plane is set where we set the previous masks 

495 diff_mask = subtraction.difference.mask 

496 

497 # science mask should be now in INJECTED 

498 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

499 

500 # template mask should be now in INJECTED_TEMPLATE 

501 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

502 

503 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int)) 

504 # The template is convolved, so the INJECTED_TEMPLATE mask plane may 

505 # include more pixels than the FAKE mask plane 

506 injTmplt_masked &= template_fake_masked 

507 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int)) 

508 

509 # Now check that detection of fakes have the correct flag for injections 

510 detectionTask = self._setup_detection() 

511 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

512 nBad = len(excludeMaskPlanes) 

513 self.assertEqual(nBad, 1) 

514 

515 output = detectionTask.run(subtraction.matchedScience, 

516 subtraction.matchedTemplate, 

517 subtraction.difference) 

518 

519 sci_refIds = [] 

520 tmpl_refIds = [] 

521 for diaSrc in output.diaSources: 

522 if diaSrc['base_PsfFlux_instFlux'] > 0: 

523 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds) 

524 self.assertTrue(diaSrc['base_PixelFlags_flag_injected']) 

525 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter']) 

526 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template']) 

527 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

528 else: 

529 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds) 

530 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template']) 

531 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

532 self.assertFalse(diaSrc['base_PixelFlags_flag_injected']) 

533 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter']) 

534 

535 

536class DetectAndMeasureScoreTest(DetectAndMeasureTestBase, lsst.utils.tests.TestCase): 

537 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask 

538 

539 def test_detection_xy0(self): 

540 """Basic functionality test with non-zero x0 and y0. 

541 """ 

542 # Set up the simulated images 

543 noiseLevel = 1. 

544 staticSeed = 1 

545 fluxLevel = 500 

546 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

547 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

548 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

549 difference = science.clone() 

550 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

551 scienceKernel = science.psf.getKernel() 

552 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

553 

554 # Configure the detection Task 

555 detectionTask = self._setup_detection() 

556 

557 # Run detection and check the results 

558 output = detectionTask.run(science, matchedTemplate, difference, score, 

559 idFactory=self.idGenerator.make_table_id_factory()) 

560 

561 # Catalog ids should be very large from this id generator. 

562 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

563 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

564 

565 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

566 

567 def test_measurements_finite(self): 

568 """Measured fluxes and centroids should always be finite. 

569 """ 

570 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

571 

572 # Set up the simulated images 

573 noiseLevel = 1. 

574 staticSeed = 1 

575 transientSeed = 6 

576 xSize = 256 

577 ySize = 256 

578 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

579 "xSize": xSize, "ySize": ySize} 

580 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

581 nSrc=1, **kwargs) 

582 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

583 nSrc=1, **kwargs) 

584 rng = np.random.RandomState(3) 

585 xLoc = np.arange(-5, xSize+5, 10) 

586 rng.shuffle(xLoc) 

587 yLoc = np.arange(-5, ySize+5, 10) 

588 rng.shuffle(yLoc) 

589 transients, transientSources = makeTestImage(seed=transientSeed, 

590 nSrc=len(xLoc), fluxLevel=1000., 

591 noiseLevel=noiseLevel, noiseSeed=8, 

592 xLoc=xLoc, yLoc=yLoc, 

593 **kwargs) 

594 difference = science.clone() 

595 difference.maskedImage -= matchedTemplate.maskedImage 

596 difference.maskedImage += transients.maskedImage 

597 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

598 scienceKernel = science.psf.getKernel() 

599 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

600 

601 # Configure the detection Task 

602 detectionTask = self._setup_detection(doForcedMeasurement=True) 

603 

604 # Run detection and check the results 

605 output = detectionTask.run(science, matchedTemplate, difference, score) 

606 

607 for column in columnNames: 

608 self._check_values(output.diaSources[column]) 

609 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

610 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

611 self._check_values(output.diaSources.getPsfInstFlux()) 

612 

613 def test_detect_transients(self): 

614 """Run detection on a difference image containing transients. 

615 """ 

616 # Set up the simulated images 

617 noiseLevel = 1. 

618 staticSeed = 1 

619 transientSeed = 6 

620 fluxLevel = 500 

621 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

622 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

623 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

624 scienceKernel = science.psf.getKernel() 

625 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

626 

627 # Configure the detection Task 

628 detectionTask = self._setup_detection(doMerge=False) 

629 kwargs["seed"] = transientSeed 

630 kwargs["nSrc"] = 10 

631 kwargs["fluxLevel"] = 1000 

632 

633 # Run detection and check the results 

634 def _detection_wrapper(positive=True): 

635 """Simulate positive or negative transients and run detection. 

636 

637 Parameters 

638 ---------- 

639 positive : `bool`, optional 

640 If set, use positive transient sources. 

641 """ 

642 

643 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

644 difference = science.clone() 

645 difference.maskedImage -= matchedTemplate.maskedImage 

646 if positive: 

647 difference.maskedImage += transients.maskedImage 

648 else: 

649 difference.maskedImage -= transients.maskedImage 

650 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

651 # NOTE: NoiseReplacer (run by forcedMeasurement) can modify the 

652 # science image if we've e.g. removed parents post-deblending. 

653 # Pass a clone of the science image, so that it doesn't disrupt 

654 # later tests. 

655 output = detectionTask.run(science.clone(), matchedTemplate, difference, score) 

656 refIds = [] 

657 scale = 1. if positive else -1. 

658 # sources near the edge may have untrustworthy centroids 

659 goodSrcFlags = ~output.diaSources['base_PixelFlags_flag_edge'] 

660 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

661 if goodSrcFlag: 

662 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

663 _detection_wrapper(positive=True) 

664 _detection_wrapper(positive=False) 

665 

666 def test_detect_dipoles(self): 

667 """Run detection on a difference image containing dipoles. 

668 """ 

669 # Set up the simulated images 

670 noiseLevel = 1. 

671 staticSeed = 1 

672 fluxLevel = 1000 

673 fluxRange = 1.5 

674 nSources = 10 

675 offset = 1 

676 xSize = 300 

677 ySize = 300 

678 kernelSize = 32 

679 # Avoid placing sources near the edge for this test, so that we can 

680 # easily check that the correct number of sources are detected. 

681 templateBorderSize = kernelSize//2 

682 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

683 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

684 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

685 "xSize": xSize, "ySize": ySize} 

686 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

687 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

688 difference = science.clone() 

689 # Shift the template by a pixel in order to make dipoles in the difference image. 

690 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

691 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

692 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

693 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

694 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

695 scienceKernel = science.psf.getKernel() 

696 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

697 

698 detectionTask = self._setup_detection() 

699 output = detectionTask.run(science, matchedTemplate, difference, score) 

700 self.assertEqual(len(output.diaSources), len(sources)) 

701 refIds = [] 

702 for diaSource in output.diaSources: 

703 if diaSource[dipoleFlag]: 

704 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

705 rtol=0.05, atol=None, usePsfFlux=False) 

706 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

707 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

708 else: 

709 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

710 

711 def test_sky_sources(self): 

712 """Add sky sources and check that they are sufficiently far from other 

713 sources and have negligible flux. 

714 """ 

715 # Set up the simulated images 

716 noiseLevel = 1. 

717 staticSeed = 1 

718 transientSeed = 6 

719 transientFluxLevel = 1000. 

720 transientFluxRange = 1.5 

721 fluxLevel = 500 

722 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

723 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

724 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

725 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

726 nSrc=10, fluxLevel=transientFluxLevel, 

727 fluxRange=transientFluxRange, 

728 noiseLevel=noiseLevel, noiseSeed=8) 

729 difference = science.clone() 

730 difference.maskedImage -= matchedTemplate.maskedImage 

731 difference.maskedImage += transients.maskedImage 

732 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

733 scienceKernel = science.psf.getKernel() 

734 kernelWidth = np.max(scienceKernel.getDimensions())//2 

735 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

736 

737 # Configure the detection Task 

738 detectionTask = self._setup_detection(doSkySources=True) 

739 

740 # Run detection and check the results 

741 output = detectionTask.run(science, matchedTemplate, difference, score, 

742 idFactory=self.idGenerator.make_table_id_factory()) 

743 nSkySourcesGenerated = detectionTask.metadata["n_skySources"] 

744 skySources = output.diaSources[output.diaSources["sky_source"]] 

745 self.assertEqual(len(skySources), nSkySourcesGenerated) 

746 for skySource in skySources: 

747 # The sky sources should not be close to any other source 

748 with self.assertRaises(AssertionError): 

749 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

750 with self.assertRaises(AssertionError): 

751 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

752 # The sky sources should have low flux levels. 

753 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

754 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

755 

756 # Catalog ids should be very large from this id generator. 

757 self.assertTrue(all(output.diaSources['id'] > 1000000000)) 

758 

759 def test_edge_detections(self): 

760 """Sources with certain bad mask planes set should not be detected. 

761 """ 

762 # Set up the simulated images 

763 noiseLevel = 1. 

764 staticSeed = 1 

765 transientSeed = 6 

766 fluxLevel = 500 

767 radius = 2 

768 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

769 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

770 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

771 

772 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

773 scienceKernel = science.psf.getKernel() 

774 # Configure the detection Task 

775 detectionTask = self._setup_detection() 

776 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

777 nBad = len(excludeMaskPlanes) 

778 self.assertGreater(nBad, 0) 

779 kwargs["seed"] = transientSeed 

780 kwargs["nSrc"] = nBad 

781 kwargs["fluxLevel"] = 1000 

782 

783 # Run detection and check the results 

784 def _detection_wrapper(setFlags=True): 

785 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

786 difference = science.clone() 

787 difference.maskedImage -= matchedTemplate.maskedImage 

788 difference.maskedImage += transients.maskedImage 

789 if setFlags: 

790 for src, badMask in zip(transientSources, excludeMaskPlanes): 

791 srcX = int(src.getX()) 

792 srcY = int(src.getY()) 

793 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

794 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

795 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

796 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

797 output = detectionTask.run(science, matchedTemplate, difference, score) 

798 refIds = [] 

799 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes) 

800 if setFlags: 

801 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

802 else: 

803 self.assertEqual(np.sum(~goodSrcFlags), 0) 

804 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

805 if ~goodSrcFlag: 

806 with self.assertRaises(AssertionError): 

807 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

808 else: 

809 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

810 _detection_wrapper(setFlags=False) 

811 _detection_wrapper(setFlags=True) 

812 

813 

814def setup_module(module): 

815 lsst.utils.tests.init() 

816 

817 

818class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

819 pass 

820 

821 

822if __name__ == "__main__": 822 ↛ 823line 822 didn't jump to line 823, because the condition on line 822 was never true

823 lsst.utils.tests.init() 

824 unittest.main()