Coverage for tests/test_sourceSelector.py: 13%

290 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-06 04:25 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24import unittest 

25import numpy as np 

26import astropy.units as u 

27 

28import lsst.afw.table 

29import lsst.meas.algorithms 

30import lsst.meas.base.tests 

31import lsst.pipe.base 

32import lsst.utils.tests 

33 

34from lsst.meas.algorithms import ColorLimit 

35 

36 

37class SourceSelectorTester: 

38 """Mixin for testing 

39 

40 This provides a base class for doing tests common to the 

41 ScienceSourceSelectorTask and ReferenceSourceSelectorTask. 

42 """ 

43 Task = None 

44 

45 def setUp(self): 

46 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

47 schema.addField("flux", float, "Flux value") 

48 schema.addField("flux_flag", "Flag", "Bad flux?") 

49 schema.addField("other_flux", float, "Flux value 2") 

50 schema.addField("other_flux_flag", "Flag", "Bad flux 2?") 

51 schema.addField("other_fluxErr", float, "Flux error 2") 

52 schema.addField("goodFlag", "Flag", "Flagged if good") 

53 schema.addField("badFlag", "Flag", "Flagged if bad") 

54 schema.addField("starGalaxy", float, "0=star, 1=galaxy") 

55 schema.addField("nChild", np.int32, "Number of children") 

56 schema.addField("detect_isPrimary", "Flag", "Is primary detection?") 

57 schema.addField("sky_source", "Flag", "Empty sky region.") 

58 self.catalog = lsst.afw.table.SourceCatalog(schema) 

59 self.catalog.reserve(10) 

60 self.config = self.Task.ConfigClass() 

61 

62 def tearDown(self): 

63 del self.catalog 

64 

65 def check(self, expected): 

66 task = self.Task(config=self.config) 

67 results = task.run(self.catalog) 

68 self.assertListEqual(results.selected.tolist(), expected) 

69 self.assertListEqual([src.getId() for src in results.sourceCat], 

70 [src.getId() for src, ok in zip(self.catalog, expected) if ok]) 

71 

72 # Check with pandas.DataFrame version of catalog 

73 results = task.run(self.catalog.asAstropy().to_pandas()) 

74 self.assertListEqual(results.selected.tolist(), expected) 

75 self.assertListEqual(list(results.sourceCat['id']), 

76 [src.getId() for src, ok in zip(self.catalog, expected) if ok]) 

77 

78 # Check with astropy.table.Table version of catalog 

79 results = task.run(self.catalog.asAstropy()) 

80 self.assertListEqual(results.selected.tolist(), expected) 

81 self.assertListEqual(list(results.sourceCat['id']), 

82 [src.getId() for src, ok in zip(self.catalog, expected) if ok]) 

83 

84 def testFlags(self): 

85 bad1 = self.catalog.addNew() 

86 bad1.set("goodFlag", False) 

87 bad1.set("badFlag", False) 

88 bad2 = self.catalog.addNew() 

89 bad2.set("goodFlag", True) 

90 bad2.set("badFlag", True) 

91 bad3 = self.catalog.addNew() 

92 bad3.set("goodFlag", False) 

93 bad3.set("badFlag", True) 

94 good = self.catalog.addNew() 

95 good.set("goodFlag", True) 

96 good.set("badFlag", False) 

97 self.catalog["flux"] = 1.0 

98 self.catalog["other_flux"] = 1.0 

99 self.config.flags.good = ["goodFlag"] 

100 self.config.flags.bad = ["badFlag"] 

101 self.check([False, False, False, True]) 

102 

103 def testSignalToNoise(self): 

104 low = self.catalog.addNew() 

105 low.set("other_flux", 1.0) 

106 low.set("other_fluxErr", 1.0) 

107 good = self.catalog.addNew() 

108 good.set("other_flux", 1.0) 

109 good.set("other_fluxErr", 0.1) 

110 high = self.catalog.addNew() 

111 high.set("other_flux", 1.0) 

112 high.set("other_fluxErr", 0.001) 

113 self.config.doSignalToNoise = True 

114 self.config.signalToNoise.fluxField = "other_flux" 

115 self.config.signalToNoise.errField = "other_fluxErr" 

116 self.config.signalToNoise.minimum = 5.0 

117 self.config.signalToNoise.maximum = 100.0 

118 self.check([False, True, False]) 

119 

120 

121class ScienceSourceSelectorTaskTest(SourceSelectorTester, lsst.utils.tests.TestCase): 

122 Task = lsst.meas.algorithms.ScienceSourceSelectorTask 

123 

124 def setUp(self): 

125 SourceSelectorTester.setUp(self) 

126 self.config.fluxLimit.fluxField = "flux" 

127 self.config.flags.bad = [] 

128 self.config.doFluxLimit = True 

129 self.config.doFlags = True 

130 self.config.doUnresolved = False 

131 self.config.doIsolated = False 

132 

133 def testFluxLimit(self): 

134 tooBright = self.catalog.addNew() 

135 tooBright.set("flux", 1.0e10) 

136 tooBright.set("flux_flag", False) 

137 good = self.catalog.addNew() 

138 good.set("flux", 1000.0) 

139 good.set("flux_flag", False) 

140 bad = self.catalog.addNew() 

141 bad.set("flux", good.get("flux")) 

142 bad.set("flux_flag", True) 

143 tooFaint = self.catalog.addNew() 

144 tooFaint.set("flux", 1.0) 

145 tooFaint.set("flux_flag", False) 

146 self.config.fluxLimit.minimum = 10.0 

147 self.config.fluxLimit.maximum = 1.0e6 

148 self.config.fluxLimit.fluxField = "flux" 

149 self.check([False, True, False, False]) 

150 

151 # Works with no maximum set? 

152 self.config.fluxLimit.maximum = None 

153 self.check([True, True, False, False]) 

154 

155 # Works with no minimum set? 

156 self.config.fluxLimit.minimum = None 

157 self.check([True, True, False, True]) 

158 

159 def testUnresolved(self): 

160 num = 5 

161 for _ in range(num): 

162 self.catalog.addNew() 

163 self.catalog["flux"] = 1.0 

164 starGalaxy = np.linspace(0.0, 1.0, num, False) 

165 self.catalog["starGalaxy"] = starGalaxy 

166 self.config.doUnresolved = True 

167 self.config.unresolved.name = "starGalaxy" 

168 minimum, maximum = 0.3, 0.7 

169 self.config.unresolved.minimum = minimum 

170 self.config.unresolved.maximum = maximum 

171 self.check(((starGalaxy > minimum) & (starGalaxy < maximum)).tolist()) 

172 

173 # Works with no minimum set? 

174 self.config.unresolved.minimum = None 

175 self.check((starGalaxy < maximum).tolist()) 

176 

177 # Works with no maximum set? 

178 self.config.unresolved.minimum = minimum 

179 self.config.unresolved.maximum = None 

180 self.check((starGalaxy > minimum).tolist()) 

181 

182 def testIsolated(self): 

183 num = 5 

184 for _ in range(num): 

185 self.catalog.addNew() 

186 self.catalog["flux"] = 1.0 

187 parent = np.array([0, 0, 10, 0, 0], dtype=int) 

188 nChild = np.array([2, 0, 0, 0, 0], dtype=int) 

189 self.catalog["parent"] = parent 

190 self.catalog["nChild"] = nChild 

191 self.config.doIsolated = True 

192 self.config.isolated.parentName = "parent" 

193 self.config.isolated.nChildName = "nChild" 

194 self.check(((parent == 0) & (nChild == 0)).tolist()) 

195 

196 def testRequireFiniteRaDec(self): 

197 num = 5 

198 for _ in range(num): 

199 self.catalog.addNew() 

200 ra = np.array([np.nan, np.nan, 0, 0, 0], dtype=float) 

201 dec = np.array([2, np.nan, 0, 0, np.nan], dtype=float) 

202 self.catalog["coord_ra"] = ra 

203 self.catalog["coord_dec"] = dec 

204 self.config.doRequireFiniteRaDec = True 

205 self.config.requireFiniteRaDec.raColName = "coord_ra" 

206 self.config.requireFiniteRaDec.decColName = "coord_dec" 

207 self.check((np.isfinite(ra) & np.isfinite(dec)).tolist()) 

208 

209 def testRequirePrimary(self): 

210 num = 5 

211 for _ in range(num): 

212 self.catalog.addNew() 

213 primary = np.array([True, True, False, True, False], dtype=bool) 

214 self.catalog["detect_isPrimary"] = primary 

215 self.config.doRequirePrimary = True 

216 self.config.requirePrimary.primaryColName = "detect_isPrimary" 

217 self.check(primary.tolist()) 

218 

219 def testSkySource(self): 

220 num = 5 

221 for _ in range(num): 

222 self.catalog.addNew() 

223 sky = np.array([True, True, False, True, False], dtype=bool) 

224 self.catalog["sky_source"] = sky 

225 # This is a union, not an intersection, so include another selection 

226 # that would otherwise reject everything. 

227 self.config.doRequirePrimary = True 

228 self.config.doSkySources = True 

229 self.check(sky.tolist()) 

230 

231 

232class ReferenceSourceSelectorTaskTest(SourceSelectorTester, lsst.utils.tests.TestCase): 

233 Task = lsst.meas.algorithms.ReferenceSourceSelectorTask 

234 

235 def setUp(self): 

236 SourceSelectorTester.setUp(self) 

237 self.config.magLimit.fluxField = "flux" 

238 self.config.doMagLimit = True 

239 self.config.doFlags = True 

240 self.config.doUnresolved = False 

241 self.config.doRequireFiniteRaDec = False 

242 

243 def testMagnitudeLimit(self): 

244 tooBright = self.catalog.addNew() 

245 tooBright.set("flux", 1.0e10) 

246 tooBright.set("flux_flag", False) 

247 good = self.catalog.addNew() 

248 good.set("flux", 1000.0) 

249 good.set("flux_flag", False) 

250 bad = self.catalog.addNew() 

251 bad.set("flux", good.get("flux")) 

252 bad.set("flux_flag", True) 

253 tooFaint = self.catalog.addNew() 

254 tooFaint.set("flux", 1.0) 

255 tooFaint.set("flux_flag", False) 

256 # Note: magnitudes are backwards, so the minimum flux is the maximum magnitude 

257 self.config.magLimit.minimum = (1.0e6*u.nJy).to_value(u.ABmag) 

258 self.config.magLimit.maximum = (10.0*u.nJy).to_value(u.ABmag) 

259 self.config.magLimit.fluxField = "flux" 

260 self.check([False, True, False, False]) 

261 

262 # Works with no minimum set? 

263 self.config.magLimit.minimum = None 

264 self.check([True, True, False, False]) 

265 

266 # Works with no maximum set? 

267 self.config.magLimit.maximum = None 

268 self.check([True, True, False, True]) 

269 

270 def testMagErrorLimit(self): 

271 # Using an arbitrary field as if it was a magnitude error to save adding a new field 

272 field = "other_fluxErr" 

273 tooFaint = self.catalog.addNew() 

274 tooFaint.set(field, 0.5) 

275 tooBright = self.catalog.addNew() 

276 tooBright.set(field, 0.00001) 

277 good = self.catalog.addNew() 

278 good.set(field, 0.2) 

279 

280 self.config.doMagError = True 

281 self.config.magError.minimum = 0.01 

282 self.config.magError.maximum = 0.3 

283 self.config.magError.magErrField = field 

284 self.check([False, False, True]) 

285 

286 def testColorLimits(self): 

287 num = 10 

288 for _ in range(num): 

289 self.catalog.addNew() 

290 color = np.linspace(-0.5, 0.5, num, True) 

291 flux = 1000.0*u.nJy 

292 # Definition: color = mag(flux) - mag(otherFlux) 

293 otherFlux = (flux.to(u.ABmag) - color*u.mag).to_value(u.nJy) 

294 self.catalog["flux"] = flux.value 

295 self.catalog["other_flux"] = otherFlux 

296 minimum, maximum = -0.1, 0.2 

297 self.config.colorLimits = {"test": ColorLimit(primary="flux", secondary="other_flux", 

298 minimum=minimum, maximum=maximum)} 

299 self.check(((color > minimum) & (color < maximum)).tolist()) 

300 

301 # Works with no minimum set? 

302 self.config.colorLimits["test"].minimum = None 

303 self.check((color < maximum).tolist()) 

304 

305 # Works with no maximum set? 

306 self.config.colorLimits["test"].maximum = None 

307 self.config.colorLimits["test"].minimum = minimum 

308 self.check((color > minimum).tolist()) 

309 

310 # Multiple limits 

311 self.config.colorLimits = {"test": ColorLimit(primary="flux", secondary="other_flux", 

312 minimum=minimum), 

313 "other": ColorLimit(primary="flux", secondary="other_flux", 

314 maximum=maximum)} 

315 assert maximum > minimum # To be non-mutually-exclusive 

316 self.check(((color > minimum) & (color < maximum)).tolist()) 

317 

318 # Multiple mutually-exclusive limits 

319 self.config.colorLimits["test"] = ColorLimit(primary="flux", secondary="other_flux", maximum=-0.1) 

320 self.config.colorLimits["other"] = ColorLimit(primary="flux", secondary="other_flux", minimum=0.1) 

321 self.check([False]*num) 

322 

323 def testUnresolved(self): 

324 num = 5 

325 for _ in range(num): 

326 self.catalog.addNew() 

327 self.catalog["flux"] = 1.0 

328 starGalaxy = np.linspace(0.0, 1.0, num, False) 

329 self.catalog["starGalaxy"] = starGalaxy 

330 self.config.doUnresolved = True 

331 self.config.unresolved.name = "starGalaxy" 

332 minimum, maximum = 0.3, 0.7 

333 self.config.unresolved.minimum = minimum 

334 self.config.unresolved.maximum = maximum 

335 self.check(((starGalaxy > minimum) & (starGalaxy < maximum)).tolist()) 

336 

337 # Works with no minimum set? 

338 self.config.unresolved.minimum = None 

339 self.check((starGalaxy < maximum).tolist()) 

340 

341 # Works with no maximum set? 

342 self.config.unresolved.minimum = minimum 

343 self.config.unresolved.maximum = None 

344 self.check((starGalaxy > minimum).tolist()) 

345 

346 def testFiniteRaDec(self): 

347 "Test that non-finite RA and Dec values are caught." 

348 num = 5 

349 for _ in range(num): 

350 self.catalog.addNew() 

351 self.catalog["coord_ra"][:] = 1.0 

352 self.catalog["coord_dec"][:] = 1.0 

353 self.catalog["coord_ra"][0] = np.nan 

354 self.catalog["coord_dec"][1] = np.inf 

355 self.config.doRequireFiniteRaDec = True 

356 

357 self.check([False, False, True, True, True]) 

358 

359 

360class TestBaseSourceSelector(lsst.utils.tests.TestCase): 

361 """Test the API of the Abstract Base Class with a trivial example.""" 

362 def setUp(self): 

363 schema = lsst.meas.base.tests.TestDataset.makeMinimalSchema() 

364 self.selectedKeyName = "is_selected" 

365 schema.addField(self.selectedKeyName, type="Flag") 

366 self.catalog = lsst.afw.table.SourceCatalog(schema) 

367 for i in range(4): 

368 self.catalog.addNew() 

369 

370 self.sourceSelector = lsst.meas.algorithms.NullSourceSelectorTask() 

371 

372 def testRun(self): 

373 """Test that run() returns a catalog and boolean selected array.""" 

374 result = self.sourceSelector.run(self.catalog) 

375 for i, x in enumerate(self.catalog['id']): 

376 self.assertIn(x, result.sourceCat['id']) 

377 self.assertTrue(result.selected[i]) 

378 

379 def testRunSourceSelectedField(self): 

380 """Test that the selected flag is set in the original catalog.""" 

381 self.sourceSelector.run(self.catalog, sourceSelectedField=self.selectedKeyName) 

382 np.testing.assert_array_equal(self.catalog[self.selectedKeyName], True) 

383 

384 def testRunNonContiguousRaises(self): 

385 """Cannot do source selection on non-contiguous catalogs.""" 

386 del self.catalog[1] # take one out of the middle to make it non-contiguous. 

387 self.assertFalse(self.catalog.isContiguous(), "Catalog is contiguous: the test won't work.") 

388 

389 with self.assertRaises(RuntimeError): 

390 self.sourceSelector.run(self.catalog) 

391 

392 

393class TestMemory(lsst.utils.tests.MemoryTestCase): 

394 pass 

395 

396 

397def setup_module(module): 

398 lsst.utils.tests.init() 

399 

400 

401if __name__ == "__main__": 401 ↛ 402line 401 didn't jump to line 402, because the condition on line 401 was never true

402 import sys 

403 setup_module(sys.modules[__name__]) 

404 unittest.main()