Coverage for tests/test_detectAndMeasure.py: 6%

489 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-01 14:05 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import unittest 

24 

25import lsst.geom 

26from lsst.ip.diffim import detectAndMeasure, subtractImages 

27from lsst.ip.diffim.utils import makeTestImage 

28from lsst.pipe.base import InvalidQuantumError 

29import lsst.utils.tests 

30 

31 

32class DetectAndMeasureTestBase(lsst.utils.tests.TestCase): 

33 

34 def _check_diaSource(self, refSources, diaSource, refIds=None, 

35 matchDistance=1., scale=1., usePsfFlux=True, 

36 rtol=0.02, atol=None): 

37 """Match a diaSource with a source in a reference catalog 

38 and compare properties. 

39 

40 Parameters 

41 ---------- 

42 refSources : `lsst.afw.table.SourceCatalog` 

43 The reference catalog. 

44 diaSource : `lsst.afw.table.SourceRecord` 

45 The new diaSource to match to the reference catalog. 

46 refIds : `list` of `int`, optional 

47 Source IDs of previously associated diaSources. 

48 matchDistance : `float`, optional 

49 Maximum distance allowed between the detected and reference source 

50 locations, in pixels. 

51 scale : `float`, optional 

52 Optional factor to scale the flux by before performing the test. 

53 usePsfFlux : `bool`, optional 

54 If set, test the PsfInstFlux field, otherwise use ApInstFlux. 

55 rtol : `float`, optional 

56 Relative tolerance of the flux value test. 

57 atol : `float`, optional 

58 Absolute tolerance of the flux value test. 

59 """ 

60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2 

61 + (diaSource.getY() - refSources.getY())**2) 

62 self.assertLess(min(distance), matchDistance) 

63 src = refSources[np.argmin(distance)] 

64 if refIds is not None: 

65 # Check that the same source was not previously associated 

66 self.assertNotIn(src.getId(), refIds) 

67 refIds.append(src.getId()) 

68 if atol is None: 

69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux() 

70 if usePsfFlux: 

71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(), 

72 rtol=rtol, atol=atol) 

73 else: 

74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(), 

75 rtol=rtol, atol=atol) 

76 

77 def _check_values(self, values, minValue=None, maxValue=None): 

78 """Verify that an array has finite values, and optionally that they are 

79 within specified minimum and maximum bounds. 

80 

81 Parameters 

82 ---------- 

83 values : `numpy.ndarray` 

84 Array of values to check. 

85 minValue : `float`, optional 

86 Minimum allowable value. 

87 maxValue : `float`, optional 

88 Maximum allowable value. 

89 """ 

90 self.assertTrue(np.all(np.isfinite(values))) 

91 if minValue is not None: 

92 self.assertTrue(np.all(values >= minValue)) 

93 if maxValue is not None: 

94 self.assertTrue(np.all(values <= maxValue)) 

95 

96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs): 

97 """Setup and configure the detection and measurement PipelineTask. 

98 

99 Parameters 

100 ---------- 

101 doSkySources : `bool`, optional 

102 Generate sky sources. 

103 nSkySources : `int`, optional 

104 The number of sky sources to add in isolated background regions. 

105 **kwargs 

106 Any additional config parameters to set. 

107 

108 Returns 

109 ------- 

110 `lsst.pipe.base.PipelineTask` 

111 The configured Task to use for detection and measurement. 

112 """ 

113 config = self.detectionTask.ConfigClass() 

114 config.doSkySources = doSkySources 

115 if doSkySources: 

116 config.skySources.nSources = nSkySources 

117 config.update(**kwargs) 

118 return self.detectionTask(config=config) 

119 

120 

121class DetectAndMeasureTest(DetectAndMeasureTestBase): 

122 detectionTask = detectAndMeasure.DetectAndMeasureTask 

123 

124 def test_detection_xy0(self): 

125 """Basic functionality test with non-zero x0 and y0. 

126 """ 

127 # Set up the simulated images 

128 noiseLevel = 1. 

129 staticSeed = 1 

130 fluxLevel = 500 

131 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

132 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

133 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

134 difference = science.clone() 

135 

136 # Configure the detection Task 

137 detectionTask = self._setup_detection() 

138 

139 # Run detection and check the results 

140 output = detectionTask.run(science, matchedTemplate, difference) 

141 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

142 

143 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

144 

145 def test_measurements_finite(self): 

146 """Measured fluxes and centroids should always be finite. 

147 """ 

148 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

149 

150 # Set up the simulated images 

151 noiseLevel = 1. 

152 staticSeed = 1 

153 transientSeed = 6 

154 xSize = 256 

155 ySize = 256 

156 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

157 "xSize": xSize, "ySize": ySize} 

158 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

159 nSrc=1, **kwargs) 

160 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

161 nSrc=1, **kwargs) 

162 rng = np.random.RandomState(3) 

163 xLoc = np.arange(-5, xSize+5, 10) 

164 rng.shuffle(xLoc) 

165 yLoc = np.arange(-5, ySize+5, 10) 

166 rng.shuffle(yLoc) 

167 transients, transientSources = makeTestImage(seed=transientSeed, 

168 nSrc=len(xLoc), fluxLevel=1000., 

169 noiseLevel=noiseLevel, noiseSeed=8, 

170 xLoc=xLoc, yLoc=yLoc, 

171 **kwargs) 

172 difference = science.clone() 

173 difference.maskedImage -= matchedTemplate.maskedImage 

174 difference.maskedImage += transients.maskedImage 

175 

176 # Configure the detection Task 

177 detectionTask = self._setup_detection(doForcedMeasurement=True) 

178 

179 # Run detection and check the results 

180 output = detectionTask.run(science, matchedTemplate, difference) 

181 

182 for column in columnNames: 

183 self._check_values(output.diaSources[column]) 

184 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

185 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

186 self._check_values(output.diaSources.getPsfInstFlux()) 

187 

188 def test_raise_config_schema_mismatch(self): 

189 """Check that sources with specified flags are removed from the catalog. 

190 """ 

191 # Configure the detection Task, and and set a config that is not in the schema 

192 with self.assertRaises(InvalidQuantumError): 

193 self._setup_detection(badSourceFlags=["Bogus_flag_42"]) 

194 

195 def test_remove_unphysical(self): 

196 """Check that sources with specified flags are removed from the catalog. 

197 """ 

198 # Set up the simulated images 

199 noiseLevel = 1. 

200 staticSeed = 1 

201 xSize = 256 

202 ySize = 256 

203 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} 

204 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

205 nSrc=1, **kwargs) 

206 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

207 nSrc=1, **kwargs) 

208 difference = science.clone() 

209 bbox = difference.getBBox() 

210 difference.maskedImage -= matchedTemplate.maskedImage 

211 

212 # Configure the detection Task, and do not remove unphysical sources 

213 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

214 badSourceFlags=[]) 

215 

216 # Run detection and check the results 

217 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

218 badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

219 nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove) 

220 # Verify that unphysical sources exist 

221 self.assertGreater(nBadNoRemove, 0) 

222 

223 # Configure the detection Task, and remove unphysical sources 

224 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

225 badSourceFlags=["base_PixelFlags_flag_offimage", ]) 

226 

227 # Run detection and check the results 

228 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

229 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

230 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) 

231 # Verify that all sources are physical 

232 self.assertEqual(nBadDoRemove, 0) 

233 # Set a few centroids outside the image bounding box 

234 nSetBad = 5 

235 for src in diaSources[0: nSetBad]: 

236 src["slot_Centroid_x"] += xSize 

237 src["slot_Centroid_y"] += ySize 

238 src["base_PixelFlags_flag_offimage"] = True 

239 # Verify that these sources are outside the image 

240 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

241 nBad = np.count_nonzero(badDiaSrc) 

242 self.assertEqual(nBad, nSetBad) 

243 diaSourcesNoBad = detectionTask._removeBadSources(diaSources) 

244 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) 

245 

246 # Verify that no sources outside the image bounding box remain 

247 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) 

248 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) 

249 

250 def test_detect_transients(self): 

251 """Run detection on a difference image containing transients. 

252 """ 

253 # Set up the simulated images 

254 noiseLevel = 1. 

255 staticSeed = 1 

256 transientSeed = 6 

257 fluxLevel = 500 

258 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

259 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

260 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

261 

262 # Configure the detection Task 

263 detectionTask = self._setup_detection(doMerge=False) 

264 kwargs["seed"] = transientSeed 

265 kwargs["nSrc"] = 10 

266 kwargs["fluxLevel"] = 1000 

267 

268 # Run detection and check the results 

269 def _detection_wrapper(positive=True): 

270 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

271 difference = science.clone() 

272 difference.maskedImage -= matchedTemplate.maskedImage 

273 if positive: 

274 difference.maskedImage += transients.maskedImage 

275 else: 

276 difference.maskedImage -= transients.maskedImage 

277 output = detectionTask.run(science, matchedTemplate, difference) 

278 refIds = [] 

279 scale = 1. if positive else -1. 

280 for diaSource in output.diaSources: 

281 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

282 _detection_wrapper(positive=True) 

283 _detection_wrapper(positive=False) 

284 

285 def test_detect_dipoles(self): 

286 """Run detection on a difference image containing dipoles. 

287 """ 

288 # Set up the simulated images 

289 noiseLevel = 1. 

290 staticSeed = 1 

291 fluxLevel = 1000 

292 fluxRange = 1.5 

293 nSources = 10 

294 offset = 1 

295 xSize = 300 

296 ySize = 300 

297 kernelSize = 32 

298 # Avoid placing sources near the edge for this test, so that we can 

299 # easily check that the correct number of sources are detected. 

300 templateBorderSize = kernelSize//2 

301 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

302 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

303 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

304 "xSize": xSize, "ySize": ySize} 

305 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

306 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

307 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

308 difference = science.clone() 

309 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

310 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

311 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

312 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

313 

314 # Configure the detection Task 

315 detectionTask = self._setup_detection(doMerge=False) 

316 

317 # Run detection and check the results 

318 output = detectionTask.run(science, matchedTemplate, difference) 

319 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

320 nSourcesDet = len(sources) 

321 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

322 refIds = [] 

323 # The diaSource check should fail if we don't merge positive and negative footprints 

324 for diaSource in output.diaSources: 

325 with self.assertRaises(AssertionError): 

326 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

327 atol=np.sqrt(fluxRange*fluxLevel)) 

328 

329 detectionTask2 = self._setup_detection(doMerge=True) 

330 output2 = detectionTask2.run(science, matchedTemplate, difference) 

331 self.assertEqual(len(output2.diaSources), nSourcesDet) 

332 refIds = [] 

333 for diaSource in output2.diaSources: 

334 if diaSource[dipoleFlag]: 

335 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

336 rtol=0.05, atol=None, usePsfFlux=False) 

337 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

338 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

339 else: 

340 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

341 

342 def test_sky_sources(self): 

343 """Add sky sources and check that they are sufficiently far from other 

344 sources and have negligible flux. 

345 """ 

346 # Set up the simulated images 

347 noiseLevel = 1. 

348 staticSeed = 1 

349 transientSeed = 6 

350 transientFluxLevel = 1000. 

351 transientFluxRange = 1.5 

352 fluxLevel = 500 

353 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

354 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

355 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

356 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

357 nSrc=10, fluxLevel=transientFluxLevel, 

358 fluxRange=transientFluxRange, 

359 noiseLevel=noiseLevel, noiseSeed=8) 

360 difference = science.clone() 

361 difference.maskedImage -= matchedTemplate.maskedImage 

362 difference.maskedImage += transients.maskedImage 

363 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2 

364 

365 # Configure the detection Task 

366 detectionTask = self._setup_detection(doSkySources=True) 

367 

368 # Run detection and check the results 

369 output = detectionTask.run(science, matchedTemplate, difference) 

370 skySources = output.diaSources[output.diaSources["sky_source"]] 

371 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources) 

372 for skySource in skySources: 

373 # The sky sources should not be close to any other source 

374 with self.assertRaises(AssertionError): 

375 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

376 with self.assertRaises(AssertionError): 

377 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

378 # The sky sources should have low flux levels. 

379 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

380 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

381 

382 def test_edge_detections(self): 

383 """Sources with certain bad mask planes set should not be detected. 

384 """ 

385 # Set up the simulated images 

386 noiseLevel = 1. 

387 staticSeed = 1 

388 transientSeed = 6 

389 fluxLevel = 500 

390 radius = 2 

391 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

392 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

393 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

394 

395 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask 

396 # Configure the detection Task 

397 detectionTask = self._setup_detection() 

398 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

399 nBad = len(excludeMaskPlanes) 

400 self.assertGreater(nBad, 0) 

401 kwargs["seed"] = transientSeed 

402 kwargs["nSrc"] = nBad 

403 kwargs["fluxLevel"] = 1000 

404 

405 # Run detection and check the results 

406 def _detection_wrapper(setFlags=True): 

407 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

408 difference = science.clone() 

409 difference.maskedImage -= matchedTemplate.maskedImage 

410 difference.maskedImage += transients.maskedImage 

411 if setFlags: 

412 for src, badMask in zip(transientSources, excludeMaskPlanes): 

413 srcX = int(src.getX()) 

414 srcY = int(src.getY()) 

415 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

416 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

417 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

418 output = detectionTask.run(science, matchedTemplate, difference) 

419 refIds = [] 

420 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes) 

421 if setFlags: 

422 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

423 else: 

424 self.assertEqual(np.sum(~goodSrcFlags), 0) 

425 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

426 if ~goodSrcFlag: 

427 with self.assertRaises(AssertionError): 

428 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

429 else: 

430 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

431 _detection_wrapper(setFlags=False) 

432 _detection_wrapper(setFlags=True) 

433 

434 def test_fake_mask_plane_propagation(self): 

435 """Test that we have the mask planes related to fakes in diffim images. 

436 This is testing method called updateMasks 

437 """ 

438 xSize = 256 

439 ySize = 256 

440 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

441 science_fake_img, science_fake_sources = makeTestImage( 

442 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1 

443 ) 

444 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

445 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

446 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1 

447 ) 

448 # created fakes and added them to the images 

449 science.image += science_fake_img.image 

450 template.image += tmplt_fake_img.image 

451 

452 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

453 # adding mask planes to both science and template images 

454 science.mask.addMaskPlane("FAKE") 

455 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE") 

456 template.mask.addMaskPlane("FAKE") 

457 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE") 

458 

459 for a_science_source in science_fake_sources: 

460 bbox = a_science_source.getFootprint().getBBox() 

461 science[bbox].mask.array |= science_fake_bitmask 

462 

463 for a_template_source in tmplt_fake_sources: 

464 bbox = a_template_source.getFootprint().getBBox() 

465 template[bbox].mask.array |= template_fake_bitmask 

466 

467 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0 

468 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 

469 

470 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

471 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) 

472 subtraction = subtractTask.run(template, science, sources) 

473 

474 # check subtraction mask plane is set where we set the previous masks 

475 diff_mask = subtraction.difference.mask 

476 

477 # science mask should be now in INJECTED 

478 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

479 

480 # template mask should be now in INJECTED_TEMPLATE 

481 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

482 

483 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int)) 

484 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int)) 

485 

486 # Now check that detection of fakes have the correct flag for injections 

487 detectionTask = self._setup_detection() 

488 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

489 nBad = len(excludeMaskPlanes) 

490 self.assertEqual(nBad, 1) 

491 

492 output = detectionTask.run(subtraction.matchedScience, 

493 subtraction.matchedTemplate, 

494 subtraction.difference) 

495 

496 sci_refIds = [] 

497 tmpl_refIds = [] 

498 for diaSrc in output.diaSources: 

499 if diaSrc['base_PsfFlux_instFlux'] > 0: 

500 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds) 

501 self.assertTrue(diaSrc['base_PixelFlags_flag_injected']) 

502 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter']) 

503 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template']) 

504 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

505 else: 

506 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds) 

507 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template']) 

508 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

509 self.assertFalse(diaSrc['base_PixelFlags_flag_injected']) 

510 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter']) 

511 

512 

513class DetectAndMeasureScoreTest(DetectAndMeasureTestBase): 

514 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask 

515 

516 def test_detection_xy0(self): 

517 """Basic functionality test with non-zero x0 and y0. 

518 """ 

519 # Set up the simulated images 

520 noiseLevel = 1. 

521 staticSeed = 1 

522 fluxLevel = 500 

523 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

524 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

525 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

526 difference = science.clone() 

527 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

528 scienceKernel = science.psf.getKernel() 

529 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

530 

531 # Configure the detection Task 

532 detectionTask = self._setup_detection() 

533 

534 # Run detection and check the results 

535 output = detectionTask.run(science, matchedTemplate, difference, score) 

536 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

537 

538 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

539 

540 def test_measurements_finite(self): 

541 """Measured fluxes and centroids should always be finite. 

542 """ 

543 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

544 

545 # Set up the simulated images 

546 noiseLevel = 1. 

547 staticSeed = 1 

548 transientSeed = 6 

549 xSize = 256 

550 ySize = 256 

551 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

552 "xSize": xSize, "ySize": ySize} 

553 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

554 nSrc=1, **kwargs) 

555 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

556 nSrc=1, **kwargs) 

557 rng = np.random.RandomState(3) 

558 xLoc = np.arange(-5, xSize+5, 10) 

559 rng.shuffle(xLoc) 

560 yLoc = np.arange(-5, ySize+5, 10) 

561 rng.shuffle(yLoc) 

562 transients, transientSources = makeTestImage(seed=transientSeed, 

563 nSrc=len(xLoc), fluxLevel=1000., 

564 noiseLevel=noiseLevel, noiseSeed=8, 

565 xLoc=xLoc, yLoc=yLoc, 

566 **kwargs) 

567 difference = science.clone() 

568 difference.maskedImage -= matchedTemplate.maskedImage 

569 difference.maskedImage += transients.maskedImage 

570 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

571 scienceKernel = science.psf.getKernel() 

572 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

573 

574 # Configure the detection Task 

575 detectionTask = self._setup_detection(doForcedMeasurement=True) 

576 

577 # Run detection and check the results 

578 output = detectionTask.run(science, matchedTemplate, difference, score) 

579 

580 for column in columnNames: 

581 self._check_values(output.diaSources[column]) 

582 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

583 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

584 self._check_values(output.diaSources.getPsfInstFlux()) 

585 

586 def test_detect_transients(self): 

587 """Run detection on a difference image containing transients. 

588 """ 

589 # Set up the simulated images 

590 noiseLevel = 1. 

591 staticSeed = 1 

592 transientSeed = 6 

593 fluxLevel = 500 

594 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

595 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

596 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

597 scienceKernel = science.psf.getKernel() 

598 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

599 

600 # Configure the detection Task 

601 detectionTask = self._setup_detection(doMerge=False) 

602 kwargs["seed"] = transientSeed 

603 kwargs["nSrc"] = 10 

604 kwargs["fluxLevel"] = 1000 

605 

606 # Run detection and check the results 

607 def _detection_wrapper(positive=True): 

608 """Simulate positive or negative transients and run detection. 

609 

610 Parameters 

611 ---------- 

612 positive : `bool`, optional 

613 If set, use positive transient sources. 

614 """ 

615 

616 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

617 difference = science.clone() 

618 difference.maskedImage -= matchedTemplate.maskedImage 

619 if positive: 

620 difference.maskedImage += transients.maskedImage 

621 else: 

622 difference.maskedImage -= transients.maskedImage 

623 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

624 output = detectionTask.run(science, matchedTemplate, difference, score) 

625 refIds = [] 

626 scale = 1. if positive else -1. 

627 goodSrcFlags = subtractTask._checkMask(score.mask, transientSources, 

628 subtractTask.config.badMaskPlanes) 

629 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

630 if ~goodSrcFlag: 

631 with self.assertRaises(AssertionError): 

632 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

633 else: 

634 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

635 _detection_wrapper(positive=True) 

636 _detection_wrapper(positive=False) 

637 

638 def test_detect_dipoles(self): 

639 """Run detection on a difference image containing dipoles. 

640 """ 

641 # Set up the simulated images 

642 noiseLevel = 1. 

643 staticSeed = 1 

644 fluxLevel = 1000 

645 fluxRange = 1.5 

646 nSources = 10 

647 offset = 1 

648 xSize = 300 

649 ySize = 300 

650 kernelSize = 32 

651 # Avoid placing sources near the edge for this test, so that we can 

652 # easily check that the correct number of sources are detected. 

653 templateBorderSize = kernelSize//2 

654 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

655 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

656 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

657 "xSize": xSize, "ySize": ySize} 

658 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

659 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

660 difference = science.clone() 

661 # Shift the template by a pixel in order to make dipoles in the difference image. 

662 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

663 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

664 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

665 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

666 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

667 scienceKernel = science.psf.getKernel() 

668 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

669 

670 # Configure the detection Task 

671 detectionTask = self._setup_detection(doMerge=False) 

672 

673 # Run detection and check the results 

674 output = detectionTask.run(science, matchedTemplate, difference, score) 

675 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

676 nSourcesDet = len(sources) 

677 # Since we did not merge the dipoles, each source should result in 

678 # both a positive and a negative diaSource 

679 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

680 refIds = [] 

681 # The diaSource check should fail if we don't merge positive and negative footprints 

682 for diaSource in output.diaSources: 

683 with self.assertRaises(AssertionError): 

684 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

685 atol=np.sqrt(fluxRange*fluxLevel)) 

686 

687 detectionTask2 = self._setup_detection(doMerge=True) 

688 output2 = detectionTask2.run(science, matchedTemplate, difference, score) 

689 self.assertEqual(len(output2.diaSources), nSourcesDet) 

690 refIds = [] 

691 for diaSource in output2.diaSources: 

692 if diaSource[dipoleFlag]: 

693 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

694 rtol=0.05, atol=None, usePsfFlux=False) 

695 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

696 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

697 else: 

698 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

699 

700 def test_sky_sources(self): 

701 """Add sky sources and check that they are sufficiently far from other 

702 sources and have negligible flux. 

703 """ 

704 # Set up the simulated images 

705 noiseLevel = 1. 

706 staticSeed = 1 

707 transientSeed = 6 

708 transientFluxLevel = 1000. 

709 transientFluxRange = 1.5 

710 fluxLevel = 500 

711 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

712 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

713 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

714 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

715 nSrc=10, fluxLevel=transientFluxLevel, 

716 fluxRange=transientFluxRange, 

717 noiseLevel=noiseLevel, noiseSeed=8) 

718 difference = science.clone() 

719 difference.maskedImage -= matchedTemplate.maskedImage 

720 difference.maskedImage += transients.maskedImage 

721 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

722 scienceKernel = science.psf.getKernel() 

723 kernelWidth = np.max(scienceKernel.getDimensions())//2 

724 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

725 

726 # Configure the detection Task 

727 detectionTask = self._setup_detection(doSkySources=True) 

728 

729 # Run detection and check the results 

730 output = detectionTask.run(science, matchedTemplate, difference, score) 

731 nSkySourcesGenerated = detectionTask.metadata["nSkySources"] 

732 skySources = output.diaSources[output.diaSources["sky_source"]] 

733 self.assertEqual(len(skySources), nSkySourcesGenerated) 

734 for skySource in skySources: 

735 # The sky sources should not be close to any other source 

736 with self.assertRaises(AssertionError): 

737 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

738 with self.assertRaises(AssertionError): 

739 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

740 # The sky sources should have low flux levels. 

741 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

742 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

743 

744 def test_edge_detections(self): 

745 """Sources with certain bad mask planes set should not be detected. 

746 """ 

747 # Set up the simulated images 

748 noiseLevel = 1. 

749 staticSeed = 1 

750 transientSeed = 6 

751 fluxLevel = 500 

752 radius = 2 

753 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

754 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

755 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

756 

757 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

758 scienceKernel = science.psf.getKernel() 

759 # Configure the detection Task 

760 detectionTask = self._setup_detection() 

761 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

762 nBad = len(excludeMaskPlanes) 

763 self.assertGreater(nBad, 0) 

764 kwargs["seed"] = transientSeed 

765 kwargs["nSrc"] = nBad 

766 kwargs["fluxLevel"] = 1000 

767 

768 # Run detection and check the results 

769 def _detection_wrapper(setFlags=True): 

770 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

771 difference = science.clone() 

772 difference.maskedImage -= matchedTemplate.maskedImage 

773 difference.maskedImage += transients.maskedImage 

774 if setFlags: 

775 for src, badMask in zip(transientSources, excludeMaskPlanes): 

776 srcX = int(src.getX()) 

777 srcY = int(src.getY()) 

778 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

779 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

780 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

781 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

782 output = detectionTask.run(science, matchedTemplate, difference, score) 

783 refIds = [] 

784 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes) 

785 if setFlags: 

786 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

787 else: 

788 self.assertEqual(np.sum(~goodSrcFlags), 0) 

789 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

790 if ~goodSrcFlag: 

791 with self.assertRaises(AssertionError): 

792 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

793 else: 

794 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

795 _detection_wrapper(setFlags=False) 

796 _detection_wrapper(setFlags=True) 

797 

798 

799def setup_module(module): 

800 lsst.utils.tests.init() 

801 

802 

803class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

804 pass 

805 

806 

807if __name__ == "__main__": 807 ↛ 808line 807 didn't jump to line 808, because the condition on line 807 was never true

808 lsst.utils.tests.init() 

809 unittest.main()