Coverage for tests/test_detectAndMeasure.py: 6%

500 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-19 09:26 +0000

1# This file is part of ip_diffim. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import numpy as np 

23import unittest 

24 

25import lsst.geom 

26from lsst.ip.diffim import detectAndMeasure, subtractImages 

27from lsst.ip.diffim.utils import makeTestImage 

28from lsst.pipe.base import InvalidQuantumError 

29import lsst.utils.tests 

30 

31 

32class DetectAndMeasureTestBase(lsst.utils.tests.TestCase): 

33 

34 def _check_diaSource(self, refSources, diaSource, refIds=None, 

35 matchDistance=1., scale=1., usePsfFlux=True, 

36 rtol=0.02, atol=None): 

37 """Match a diaSource with a source in a reference catalog 

38 and compare properties. 

39 

40 Parameters 

41 ---------- 

42 refSources : `lsst.afw.table.SourceCatalog` 

43 The reference catalog. 

44 diaSource : `lsst.afw.table.SourceRecord` 

45 The new diaSource to match to the reference catalog. 

46 refIds : `list` of `int`, optional 

47 Source IDs of previously associated diaSources. 

48 matchDistance : `float`, optional 

49 Maximum distance allowed between the detected and reference source 

50 locations, in pixels. 

51 scale : `float`, optional 

52 Optional factor to scale the flux by before performing the test. 

53 usePsfFlux : `bool`, optional 

54 If set, test the PsfInstFlux field, otherwise use ApInstFlux. 

55 rtol : `float`, optional 

56 Relative tolerance of the flux value test. 

57 atol : `float`, optional 

58 Absolute tolerance of the flux value test. 

59 """ 

60 distance = np.sqrt((diaSource.getX() - refSources.getX())**2 

61 + (diaSource.getY() - refSources.getY())**2) 

62 self.assertLess(min(distance), matchDistance) 

63 src = refSources[np.argmin(distance)] 

64 if refIds is not None: 

65 # Check that the same source was not previously associated 

66 self.assertNotIn(src.getId(), refIds) 

67 refIds.append(src.getId()) 

68 if atol is None: 

69 atol = rtol*src.getPsfInstFlux() if usePsfFlux else rtol*src.getApInstFlux() 

70 if usePsfFlux: 

71 self.assertFloatsAlmostEqual(src.getPsfInstFlux()*scale, diaSource.getPsfInstFlux(), 

72 rtol=rtol, atol=atol) 

73 else: 

74 self.assertFloatsAlmostEqual(src.getApInstFlux()*scale, diaSource.getApInstFlux(), 

75 rtol=rtol, atol=atol) 

76 

77 def _check_values(self, values, minValue=None, maxValue=None): 

78 """Verify that an array has finite values, and optionally that they are 

79 within specified minimum and maximum bounds. 

80 

81 Parameters 

82 ---------- 

83 values : `numpy.ndarray` 

84 Array of values to check. 

85 minValue : `float`, optional 

86 Minimum allowable value. 

87 maxValue : `float`, optional 

88 Maximum allowable value. 

89 """ 

90 self.assertTrue(np.all(np.isfinite(values))) 

91 if minValue is not None: 

92 self.assertTrue(np.all(values >= minValue)) 

93 if maxValue is not None: 

94 self.assertTrue(np.all(values <= maxValue)) 

95 

96 def _setup_detection(self, doSkySources=False, nSkySources=5, **kwargs): 

97 """Setup and configure the detection and measurement PipelineTask. 

98 

99 Parameters 

100 ---------- 

101 doSkySources : `bool`, optional 

102 Generate sky sources. 

103 nSkySources : `int`, optional 

104 The number of sky sources to add in isolated background regions. 

105 **kwargs 

106 Any additional config parameters to set. 

107 

108 Returns 

109 ------- 

110 `lsst.pipe.base.PipelineTask` 

111 The configured Task to use for detection and measurement. 

112 """ 

113 config = self.detectionTask.ConfigClass() 

114 config.doSkySources = doSkySources 

115 if doSkySources: 

116 config.skySources.nSources = nSkySources 

117 config.update(**kwargs) 

118 return self.detectionTask(config=config) 

119 

120 

121class DetectAndMeasureTest(DetectAndMeasureTestBase): 

122 detectionTask = detectAndMeasure.DetectAndMeasureTask 

123 

124 def test_detection_xy0(self): 

125 """Basic functionality test with non-zero x0 and y0. 

126 """ 

127 # Set up the simulated images 

128 noiseLevel = 1. 

129 staticSeed = 1 

130 fluxLevel = 500 

131 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

132 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

133 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

134 difference = science.clone() 

135 

136 # Configure the detection Task 

137 detectionTask = self._setup_detection() 

138 

139 # Run detection and check the results 

140 output = detectionTask.run(science, matchedTemplate, difference) 

141 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

142 

143 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

144 

145 def test_measurements_finite(self): 

146 """Measured fluxes and centroids should always be finite. 

147 """ 

148 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

149 

150 # Set up the simulated images 

151 noiseLevel = 1. 

152 staticSeed = 1 

153 transientSeed = 6 

154 xSize = 256 

155 ySize = 256 

156 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

157 "xSize": xSize, "ySize": ySize} 

158 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

159 nSrc=1, **kwargs) 

160 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

161 nSrc=1, **kwargs) 

162 rng = np.random.RandomState(3) 

163 xLoc = np.arange(-5, xSize+5, 10) 

164 rng.shuffle(xLoc) 

165 yLoc = np.arange(-5, ySize+5, 10) 

166 rng.shuffle(yLoc) 

167 transients, transientSources = makeTestImage(seed=transientSeed, 

168 nSrc=len(xLoc), fluxLevel=1000., 

169 noiseLevel=noiseLevel, noiseSeed=8, 

170 xLoc=xLoc, yLoc=yLoc, 

171 **kwargs) 

172 difference = science.clone() 

173 difference.maskedImage -= matchedTemplate.maskedImage 

174 difference.maskedImage += transients.maskedImage 

175 

176 # Configure the detection Task 

177 detectionTask = self._setup_detection(doForcedMeasurement=True) 

178 

179 # Run detection and check the results 

180 output = detectionTask.run(science, matchedTemplate, difference) 

181 

182 for column in columnNames: 

183 self._check_values(output.diaSources[column]) 

184 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

185 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

186 self._check_values(output.diaSources.getPsfInstFlux()) 

187 

188 def test_raise_config_schema_mismatch(self): 

189 """Check that sources with specified flags are removed from the catalog. 

190 """ 

191 # Configure the detection Task, and and set a config that is not in the schema 

192 with self.assertRaises(InvalidQuantumError): 

193 self._setup_detection(badSourceFlags=["Bogus_flag_42"]) 

194 

195 def test_remove_unphysical(self): 

196 """Check that sources with specified flags are removed from the catalog. 

197 """ 

198 # Set up the simulated images 

199 noiseLevel = 1. 

200 staticSeed = 1 

201 xSize = 256 

202 ySize = 256 

203 kwargs = {"psfSize": 2.4, "xSize": xSize, "ySize": ySize} 

204 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

205 nSrc=1, **kwargs) 

206 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

207 nSrc=1, **kwargs) 

208 difference = science.clone() 

209 bbox = difference.getBBox() 

210 difference.maskedImage -= matchedTemplate.maskedImage 

211 

212 # Configure the detection Task, and do not remove unphysical sources 

213 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

214 badSourceFlags=[]) 

215 

216 # Run detection and check the results 

217 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

218 badDiaSrcNoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

219 nBadNoRemove = np.count_nonzero(badDiaSrcNoRemove) 

220 # Verify that unphysical sources exist 

221 self.assertGreater(nBadNoRemove, 0) 

222 

223 # Configure the detection Task, and remove unphysical sources 

224 detectionTask = self._setup_detection(doForcedMeasurement=False, doSkySources=True, nSkySources=20, 

225 badSourceFlags=["base_PixelFlags_flag_offimage", ]) 

226 

227 # Run detection and check the results 

228 diaSources = detectionTask.run(science, matchedTemplate, difference).diaSources 

229 badDiaSrcDoRemove = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

230 nBadDoRemove = np.count_nonzero(badDiaSrcDoRemove) 

231 # Verify that all sources are physical 

232 self.assertEqual(nBadDoRemove, 0) 

233 # Set a few centroids outside the image bounding box 

234 nSetBad = 5 

235 for src in diaSources[0: nSetBad]: 

236 src["slot_Centroid_x"] += xSize 

237 src["slot_Centroid_y"] += ySize 

238 src["base_PixelFlags_flag_offimage"] = True 

239 # Verify that these sources are outside the image 

240 badDiaSrc = ~bbox.contains(diaSources.getX(), diaSources.getY()) 

241 nBad = np.count_nonzero(badDiaSrc) 

242 self.assertEqual(nBad, nSetBad) 

243 diaSourcesNoBad = detectionTask._removeBadSources(diaSources) 

244 badDiaSrcNoBad = ~bbox.contains(diaSourcesNoBad.getX(), diaSourcesNoBad.getY()) 

245 

246 # Verify that no sources outside the image bounding box remain 

247 self.assertEqual(np.count_nonzero(badDiaSrcNoBad), 0) 

248 self.assertEqual(len(diaSourcesNoBad), len(diaSources) - nSetBad) 

249 

250 def test_detect_transients(self): 

251 """Run detection on a difference image containing transients. 

252 """ 

253 # Set up the simulated images 

254 noiseLevel = 1. 

255 staticSeed = 1 

256 transientSeed = 6 

257 fluxLevel = 500 

258 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

259 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

260 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

261 

262 # Configure the detection Task 

263 detectionTask = self._setup_detection(doMerge=False) 

264 kwargs["seed"] = transientSeed 

265 kwargs["nSrc"] = 10 

266 kwargs["fluxLevel"] = 1000 

267 

268 # Run detection and check the results 

269 def _detection_wrapper(positive=True): 

270 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

271 difference = science.clone() 

272 difference.maskedImage -= matchedTemplate.maskedImage 

273 if positive: 

274 difference.maskedImage += transients.maskedImage 

275 else: 

276 difference.maskedImage -= transients.maskedImage 

277 output = detectionTask.run(science, matchedTemplate, difference) 

278 refIds = [] 

279 scale = 1. if positive else -1. 

280 for diaSource in output.diaSources: 

281 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

282 _detection_wrapper(positive=True) 

283 _detection_wrapper(positive=False) 

284 

285 def test_missing_mask_planes(self): 

286 """Check that detection runs with missing mask planes. 

287 """ 

288 # Set up the simulated images 

289 noiseLevel = 1. 

290 fluxLevel = 500 

291 kwargs = {"psfSize": 2.4, "fluxLevel": fluxLevel, "addMaskPlanes": []} 

292 # Use different seeds for the science and template so every source is a diaSource 

293 science, sources = makeTestImage(seed=5, noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

294 matchedTemplate, _ = makeTestImage(seed=6, noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

295 

296 difference = science.clone() 

297 difference.maskedImage -= matchedTemplate.maskedImage 

298 detectionTask = self._setup_detection() 

299 

300 # Verify that detection runs without errors 

301 detectionTask.run(science, matchedTemplate, difference) 

302 

303 def test_detect_dipoles(self): 

304 """Run detection on a difference image containing dipoles. 

305 """ 

306 # Set up the simulated images 

307 noiseLevel = 1. 

308 staticSeed = 1 

309 fluxLevel = 1000 

310 fluxRange = 1.5 

311 nSources = 10 

312 offset = 1 

313 xSize = 300 

314 ySize = 300 

315 kernelSize = 32 

316 # Avoid placing sources near the edge for this test, so that we can 

317 # easily check that the correct number of sources are detected. 

318 templateBorderSize = kernelSize//2 

319 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

320 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

321 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

322 "xSize": xSize, "ySize": ySize} 

323 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

324 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

325 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

326 difference = science.clone() 

327 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

328 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

329 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

330 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

331 

332 # Configure the detection Task 

333 detectionTask = self._setup_detection(doMerge=False) 

334 

335 # Run detection and check the results 

336 output = detectionTask.run(science, matchedTemplate, difference) 

337 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

338 nSourcesDet = len(sources) 

339 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

340 refIds = [] 

341 # The diaSource check should fail if we don't merge positive and negative footprints 

342 for diaSource in output.diaSources: 

343 with self.assertRaises(AssertionError): 

344 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

345 atol=np.sqrt(fluxRange*fluxLevel)) 

346 

347 detectionTask2 = self._setup_detection(doMerge=True) 

348 output2 = detectionTask2.run(science, matchedTemplate, difference) 

349 self.assertEqual(len(output2.diaSources), nSourcesDet) 

350 refIds = [] 

351 for diaSource in output2.diaSources: 

352 if diaSource[dipoleFlag]: 

353 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

354 rtol=0.05, atol=None, usePsfFlux=False) 

355 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

356 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

357 else: 

358 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

359 

360 def test_sky_sources(self): 

361 """Add sky sources and check that they are sufficiently far from other 

362 sources and have negligible flux. 

363 """ 

364 # Set up the simulated images 

365 noiseLevel = 1. 

366 staticSeed = 1 

367 transientSeed = 6 

368 transientFluxLevel = 1000. 

369 transientFluxRange = 1.5 

370 fluxLevel = 500 

371 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

372 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

373 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

374 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

375 nSrc=10, fluxLevel=transientFluxLevel, 

376 fluxRange=transientFluxRange, 

377 noiseLevel=noiseLevel, noiseSeed=8) 

378 difference = science.clone() 

379 difference.maskedImage -= matchedTemplate.maskedImage 

380 difference.maskedImage += transients.maskedImage 

381 kernelWidth = np.max(science.psf.getKernel().getDimensions())//2 

382 

383 # Configure the detection Task 

384 detectionTask = self._setup_detection(doSkySources=True) 

385 

386 # Run detection and check the results 

387 output = detectionTask.run(science, matchedTemplate, difference) 

388 skySources = output.diaSources[output.diaSources["sky_source"]] 

389 self.assertEqual(len(skySources), detectionTask.config.skySources.nSources) 

390 for skySource in skySources: 

391 # The sky sources should not be close to any other source 

392 with self.assertRaises(AssertionError): 

393 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

394 with self.assertRaises(AssertionError): 

395 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

396 # The sky sources should have low flux levels. 

397 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

398 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

399 

400 def test_edge_detections(self): 

401 """Sources with certain bad mask planes set should not be detected. 

402 """ 

403 # Set up the simulated images 

404 noiseLevel = 1. 

405 staticSeed = 1 

406 transientSeed = 6 

407 fluxLevel = 500 

408 radius = 2 

409 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

410 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

411 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

412 

413 _checkMask = subtractImages.AlardLuptonSubtractTask._checkMask 

414 # Configure the detection Task 

415 detectionTask = self._setup_detection() 

416 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

417 nBad = len(excludeMaskPlanes) 

418 self.assertGreater(nBad, 0) 

419 kwargs["seed"] = transientSeed 

420 kwargs["nSrc"] = nBad 

421 kwargs["fluxLevel"] = 1000 

422 

423 # Run detection and check the results 

424 def _detection_wrapper(setFlags=True): 

425 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

426 difference = science.clone() 

427 difference.maskedImage -= matchedTemplate.maskedImage 

428 difference.maskedImage += transients.maskedImage 

429 if setFlags: 

430 for src, badMask in zip(transientSources, excludeMaskPlanes): 

431 srcX = int(src.getX()) 

432 srcY = int(src.getY()) 

433 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

434 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

435 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

436 output = detectionTask.run(science, matchedTemplate, difference) 

437 refIds = [] 

438 goodSrcFlags = _checkMask(difference.mask, transientSources, excludeMaskPlanes) 

439 if setFlags: 

440 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

441 else: 

442 self.assertEqual(np.sum(~goodSrcFlags), 0) 

443 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

444 if ~goodSrcFlag: 

445 with self.assertRaises(AssertionError): 

446 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

447 else: 

448 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

449 _detection_wrapper(setFlags=False) 

450 _detection_wrapper(setFlags=True) 

451 

452 def test_fake_mask_plane_propagation(self): 

453 """Test that we have the mask planes related to fakes in diffim images. 

454 This is testing method called updateMasks 

455 """ 

456 xSize = 256 

457 ySize = 256 

458 science, sources = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

459 science_fake_img, science_fake_sources = makeTestImage( 

460 psfSize=2.4, xSize=xSize, ySize=ySize, seed=5, nSrc=3, noiseLevel=0.25, fluxRange=1 

461 ) 

462 template, _ = makeTestImage(psfSize=2.4, xSize=xSize, ySize=ySize, doApplyCalibration=True) 

463 tmplt_fake_img, tmplt_fake_sources = makeTestImage( 

464 psfSize=2.4, xSize=xSize, ySize=ySize, seed=9, nSrc=3, noiseLevel=0.25, fluxRange=1 

465 ) 

466 # created fakes and added them to the images 

467 science.image += science_fake_img.image 

468 template.image += tmplt_fake_img.image 

469 

470 # TODO: DM-40796 update to INJECTED names when source injection gets refactored 

471 # adding mask planes to both science and template images 

472 science.mask.addMaskPlane("FAKE") 

473 science_fake_bitmask = science.mask.getPlaneBitMask("FAKE") 

474 template.mask.addMaskPlane("FAKE") 

475 template_fake_bitmask = template.mask.getPlaneBitMask("FAKE") 

476 

477 for a_science_source in science_fake_sources: 

478 bbox = a_science_source.getFootprint().getBBox() 

479 science[bbox].mask.array |= science_fake_bitmask 

480 

481 for a_template_source in tmplt_fake_sources: 

482 bbox = a_template_source.getFootprint().getBBox() 

483 template[bbox].mask.array |= template_fake_bitmask 

484 

485 science_fake_masked = (science.mask.array & science_fake_bitmask) > 0 

486 template_fake_masked = (template.mask.array & template_fake_bitmask) > 0 

487 

488 subtractConfig = subtractImages.AlardLuptonSubtractTask.ConfigClass() 

489 subtractTask = subtractImages.AlardLuptonSubtractTask(config=subtractConfig) 

490 subtraction = subtractTask.run(template, science, sources) 

491 

492 # check subtraction mask plane is set where we set the previous masks 

493 diff_mask = subtraction.difference.mask 

494 

495 # science mask should be now in INJECTED 

496 inj_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED")) > 0 

497 

498 # template mask should be now in INJECTED_TEMPLATE 

499 injTmplt_masked = (diff_mask.array & diff_mask.getPlaneBitMask("INJECTED_TEMPLATE")) > 0 

500 

501 self.assertFloatsEqual(inj_masked.astype(int), science_fake_masked.astype(int)) 

502 # The template is convolved, so the INJECTED_TEMPLATE mask plane may 

503 # include more pixels than the FAKE mask plane 

504 injTmplt_masked &= template_fake_masked 

505 self.assertFloatsEqual(injTmplt_masked.astype(int), template_fake_masked.astype(int)) 

506 

507 # Now check that detection of fakes have the correct flag for injections 

508 detectionTask = self._setup_detection() 

509 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

510 nBad = len(excludeMaskPlanes) 

511 self.assertEqual(nBad, 1) 

512 

513 output = detectionTask.run(subtraction.matchedScience, 

514 subtraction.matchedTemplate, 

515 subtraction.difference) 

516 

517 sci_refIds = [] 

518 tmpl_refIds = [] 

519 for diaSrc in output.diaSources: 

520 if diaSrc['base_PsfFlux_instFlux'] > 0: 

521 self._check_diaSource(science_fake_sources, diaSrc, scale=1, refIds=sci_refIds) 

522 self.assertTrue(diaSrc['base_PixelFlags_flag_injected']) 

523 self.assertTrue(diaSrc['base_PixelFlags_flag_injectedCenter']) 

524 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_template']) 

525 self.assertFalse(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

526 else: 

527 self._check_diaSource(tmplt_fake_sources, diaSrc, scale=-1, refIds=tmpl_refIds) 

528 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_template']) 

529 self.assertTrue(diaSrc['base_PixelFlags_flag_injected_templateCenter']) 

530 self.assertFalse(diaSrc['base_PixelFlags_flag_injected']) 

531 self.assertFalse(diaSrc['base_PixelFlags_flag_injectedCenter']) 

532 

533 

534class DetectAndMeasureScoreTest(DetectAndMeasureTestBase): 

535 detectionTask = detectAndMeasure.DetectAndMeasureScoreTask 

536 

537 def test_detection_xy0(self): 

538 """Basic functionality test with non-zero x0 and y0. 

539 """ 

540 # Set up the simulated images 

541 noiseLevel = 1. 

542 staticSeed = 1 

543 fluxLevel = 500 

544 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "x0": 12345, "y0": 67890} 

545 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

546 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

547 difference = science.clone() 

548 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

549 scienceKernel = science.psf.getKernel() 

550 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

551 

552 # Configure the detection Task 

553 detectionTask = self._setup_detection() 

554 

555 # Run detection and check the results 

556 output = detectionTask.run(science, matchedTemplate, difference, score) 

557 subtractedMeasuredExposure = output.subtractedMeasuredExposure 

558 

559 self.assertImagesEqual(subtractedMeasuredExposure.image, difference.image) 

560 

561 def test_measurements_finite(self): 

562 """Measured fluxes and centroids should always be finite. 

563 """ 

564 columnNames = ["coord_ra", "coord_dec", "ip_diffim_forced_PsfFlux_instFlux"] 

565 

566 # Set up the simulated images 

567 noiseLevel = 1. 

568 staticSeed = 1 

569 transientSeed = 6 

570 xSize = 256 

571 ySize = 256 

572 kwargs = {"psfSize": 2.4, "x0": 0, "y0": 0, 

573 "xSize": xSize, "ySize": ySize} 

574 science, sources = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel, noiseSeed=6, 

575 nSrc=1, **kwargs) 

576 matchedTemplate, _ = makeTestImage(seed=staticSeed, noiseLevel=noiseLevel/4, noiseSeed=7, 

577 nSrc=1, **kwargs) 

578 rng = np.random.RandomState(3) 

579 xLoc = np.arange(-5, xSize+5, 10) 

580 rng.shuffle(xLoc) 

581 yLoc = np.arange(-5, ySize+5, 10) 

582 rng.shuffle(yLoc) 

583 transients, transientSources = makeTestImage(seed=transientSeed, 

584 nSrc=len(xLoc), fluxLevel=1000., 

585 noiseLevel=noiseLevel, noiseSeed=8, 

586 xLoc=xLoc, yLoc=yLoc, 

587 **kwargs) 

588 difference = science.clone() 

589 difference.maskedImage -= matchedTemplate.maskedImage 

590 difference.maskedImage += transients.maskedImage 

591 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

592 scienceKernel = science.psf.getKernel() 

593 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

594 

595 # Configure the detection Task 

596 detectionTask = self._setup_detection(doForcedMeasurement=True) 

597 

598 # Run detection and check the results 

599 output = detectionTask.run(science, matchedTemplate, difference, score) 

600 

601 for column in columnNames: 

602 self._check_values(output.diaSources[column]) 

603 self._check_values(output.diaSources.getX(), minValue=0, maxValue=xSize) 

604 self._check_values(output.diaSources.getY(), minValue=0, maxValue=ySize) 

605 self._check_values(output.diaSources.getPsfInstFlux()) 

606 

607 def test_detect_transients(self): 

608 """Run detection on a difference image containing transients. 

609 """ 

610 # Set up the simulated images 

611 noiseLevel = 1. 

612 staticSeed = 1 

613 transientSeed = 6 

614 fluxLevel = 500 

615 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

616 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

617 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

618 scienceKernel = science.psf.getKernel() 

619 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

620 

621 # Configure the detection Task 

622 detectionTask = self._setup_detection(doMerge=False) 

623 kwargs["seed"] = transientSeed 

624 kwargs["nSrc"] = 10 

625 kwargs["fluxLevel"] = 1000 

626 

627 # Run detection and check the results 

628 def _detection_wrapper(positive=True): 

629 """Simulate positive or negative transients and run detection. 

630 

631 Parameters 

632 ---------- 

633 positive : `bool`, optional 

634 If set, use positive transient sources. 

635 """ 

636 

637 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

638 difference = science.clone() 

639 difference.maskedImage -= matchedTemplate.maskedImage 

640 if positive: 

641 difference.maskedImage += transients.maskedImage 

642 else: 

643 difference.maskedImage -= transients.maskedImage 

644 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

645 output = detectionTask.run(science, matchedTemplate, difference, score) 

646 refIds = [] 

647 scale = 1. if positive else -1. 

648 goodSrcFlags = subtractTask._checkMask(score.mask, transientSources, 

649 subtractTask.config.badMaskPlanes) 

650 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

651 if ~goodSrcFlag: 

652 with self.assertRaises(AssertionError): 

653 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

654 else: 

655 self._check_diaSource(transientSources, diaSource, refIds=refIds, scale=scale) 

656 _detection_wrapper(positive=True) 

657 _detection_wrapper(positive=False) 

658 

659 def test_detect_dipoles(self): 

660 """Run detection on a difference image containing dipoles. 

661 """ 

662 # Set up the simulated images 

663 noiseLevel = 1. 

664 staticSeed = 1 

665 fluxLevel = 1000 

666 fluxRange = 1.5 

667 nSources = 10 

668 offset = 1 

669 xSize = 300 

670 ySize = 300 

671 kernelSize = 32 

672 # Avoid placing sources near the edge for this test, so that we can 

673 # easily check that the correct number of sources are detected. 

674 templateBorderSize = kernelSize//2 

675 dipoleFlag = "ip_diffim_DipoleFit_flag_classification" 

676 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel, "fluxRange": fluxRange, 

677 "nSrc": nSources, "templateBorderSize": templateBorderSize, "kernelSize": kernelSize, 

678 "xSize": xSize, "ySize": ySize} 

679 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

680 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

681 difference = science.clone() 

682 # Shift the template by a pixel in order to make dipoles in the difference image. 

683 matchedTemplate.image.array[...] = np.roll(matchedTemplate.image.array[...], offset, axis=0) 

684 matchedTemplate.variance.array[...] = np.roll(matchedTemplate.variance.array[...], offset, axis=0) 

685 matchedTemplate.mask.array[...] = np.roll(matchedTemplate.mask.array[...], offset, axis=0) 

686 difference.maskedImage -= matchedTemplate.maskedImage[science.getBBox()] 

687 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

688 scienceKernel = science.psf.getKernel() 

689 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

690 

691 # Configure the detection Task 

692 detectionTask = self._setup_detection(doMerge=False) 

693 

694 # Run detection and check the results 

695 output = detectionTask.run(science, matchedTemplate, difference, score) 

696 self.assertIn(dipoleFlag, output.diaSources.schema.getNames()) 

697 nSourcesDet = len(sources) 

698 # Since we did not merge the dipoles, each source should result in 

699 # both a positive and a negative diaSource 

700 self.assertEqual(len(output.diaSources), 2*nSourcesDet) 

701 refIds = [] 

702 # The diaSource check should fail if we don't merge positive and negative footprints 

703 for diaSource in output.diaSources: 

704 with self.assertRaises(AssertionError): 

705 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

706 atol=np.sqrt(fluxRange*fluxLevel)) 

707 

708 detectionTask2 = self._setup_detection(doMerge=True) 

709 output2 = detectionTask2.run(science, matchedTemplate, difference, score) 

710 self.assertEqual(len(output2.diaSources), nSourcesDet) 

711 refIds = [] 

712 for diaSource in output2.diaSources: 

713 if diaSource[dipoleFlag]: 

714 self._check_diaSource(sources, diaSource, refIds=refIds, scale=0, 

715 rtol=0.05, atol=None, usePsfFlux=False) 

716 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_orientation"], -90., atol=2.) 

717 self.assertFloatsAlmostEqual(diaSource["ip_diffim_DipoleFit_separation"], offset, rtol=0.1) 

718 else: 

719 raise ValueError("DiaSource with ID %s is not a dipole!", diaSource.getId()) 

720 

721 def test_sky_sources(self): 

722 """Add sky sources and check that they are sufficiently far from other 

723 sources and have negligible flux. 

724 """ 

725 # Set up the simulated images 

726 noiseLevel = 1. 

727 staticSeed = 1 

728 transientSeed = 6 

729 transientFluxLevel = 1000. 

730 transientFluxRange = 1.5 

731 fluxLevel = 500 

732 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

733 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

734 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

735 transients, transientSources = makeTestImage(seed=transientSeed, psfSize=2.4, 

736 nSrc=10, fluxLevel=transientFluxLevel, 

737 fluxRange=transientFluxRange, 

738 noiseLevel=noiseLevel, noiseSeed=8) 

739 difference = science.clone() 

740 difference.maskedImage -= matchedTemplate.maskedImage 

741 difference.maskedImage += transients.maskedImage 

742 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

743 scienceKernel = science.psf.getKernel() 

744 kernelWidth = np.max(scienceKernel.getDimensions())//2 

745 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

746 

747 # Configure the detection Task 

748 detectionTask = self._setup_detection(doSkySources=True) 

749 

750 # Run detection and check the results 

751 output = detectionTask.run(science, matchedTemplate, difference, score) 

752 nSkySourcesGenerated = detectionTask.metadata["nSkySources"] 

753 skySources = output.diaSources[output.diaSources["sky_source"]] 

754 self.assertEqual(len(skySources), nSkySourcesGenerated) 

755 for skySource in skySources: 

756 # The sky sources should not be close to any other source 

757 with self.assertRaises(AssertionError): 

758 self._check_diaSource(transientSources, skySource, matchDistance=kernelWidth) 

759 with self.assertRaises(AssertionError): 

760 self._check_diaSource(sources, skySource, matchDistance=kernelWidth) 

761 # The sky sources should have low flux levels. 

762 self._check_diaSource(transientSources, skySource, matchDistance=1000, scale=0., 

763 atol=np.sqrt(transientFluxRange*transientFluxLevel)) 

764 

765 def test_edge_detections(self): 

766 """Sources with certain bad mask planes set should not be detected. 

767 """ 

768 # Set up the simulated images 

769 noiseLevel = 1. 

770 staticSeed = 1 

771 transientSeed = 6 

772 fluxLevel = 500 

773 radius = 2 

774 kwargs = {"seed": staticSeed, "psfSize": 2.4, "fluxLevel": fluxLevel} 

775 science, sources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=6, **kwargs) 

776 matchedTemplate, _ = makeTestImage(noiseLevel=noiseLevel/4, noiseSeed=7, **kwargs) 

777 

778 subtractTask = subtractImages.AlardLuptonPreconvolveSubtractTask() 

779 scienceKernel = science.psf.getKernel() 

780 # Configure the detection Task 

781 detectionTask = self._setup_detection() 

782 excludeMaskPlanes = detectionTask.config.detection.excludeMaskPlanes 

783 nBad = len(excludeMaskPlanes) 

784 self.assertGreater(nBad, 0) 

785 kwargs["seed"] = transientSeed 

786 kwargs["nSrc"] = nBad 

787 kwargs["fluxLevel"] = 1000 

788 

789 # Run detection and check the results 

790 def _detection_wrapper(setFlags=True): 

791 transients, transientSources = makeTestImage(noiseLevel=noiseLevel, noiseSeed=8, **kwargs) 

792 difference = science.clone() 

793 difference.maskedImage -= matchedTemplate.maskedImage 

794 difference.maskedImage += transients.maskedImage 

795 if setFlags: 

796 for src, badMask in zip(transientSources, excludeMaskPlanes): 

797 srcX = int(src.getX()) 

798 srcY = int(src.getY()) 

799 srcBbox = lsst.geom.Box2I(lsst.geom.Point2I(srcX - radius, srcY - radius), 

800 lsst.geom.Extent2I(2*radius + 1, 2*radius + 1)) 

801 difference[srcBbox].mask.array |= lsst.afw.image.Mask.getPlaneBitMask(badMask) 

802 score = subtractTask._convolveExposure(difference, scienceKernel, subtractTask.convolutionControl) 

803 output = detectionTask.run(science, matchedTemplate, difference, score) 

804 refIds = [] 

805 goodSrcFlags = subtractTask._checkMask(difference.mask, transientSources, excludeMaskPlanes) 

806 if setFlags: 

807 self.assertEqual(np.sum(~goodSrcFlags), nBad) 

808 else: 

809 self.assertEqual(np.sum(~goodSrcFlags), 0) 

810 for diaSource, goodSrcFlag in zip(output.diaSources, goodSrcFlags): 

811 if ~goodSrcFlag: 

812 with self.assertRaises(AssertionError): 

813 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

814 else: 

815 self._check_diaSource(transientSources, diaSource, refIds=refIds) 

816 _detection_wrapper(setFlags=False) 

817 _detection_wrapper(setFlags=True) 

818 

819 

820def setup_module(module): 

821 lsst.utils.tests.init() 

822 

823 

824class MemoryTestCase(lsst.utils.tests.MemoryTestCase): 

825 pass 

826 

827 

828if __name__ == "__main__": 828 ↛ 829line 828 didn't jump to line 829, because the condition on line 828 was never true

829 lsst.utils.tests.init() 

830 unittest.main()