Coverage for tests/test_sourceMatch.py: 12%

218 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-09-30 02:30 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for matching SourceSets 

24 

25Run with: 

26 python test_sourceMatch.py 

27or 

28 pytest test_sourceMatch.py 

29""" 

30import os 

31import re 

32import unittest 

33 

34import numpy as np 

35 

36import lsst.geom 

37import lsst.afw.table as afwTable 

38import lsst.daf.base as dafBase 

39import lsst.utils.tests 

40 

41try: 

42 afwdataDir = lsst.utils.getPackageDir("afwdata") 

43except LookupError: 

44 afwdataDir = None 

45 

46 

47class SourceMatchTestCase(lsst.utils.tests.TestCase): 

48 """A test case for matching SourceSets 

49 """ 

50 

51 def setUp(self): 

52 schema = afwTable.SourceTable.makeMinimalSchema() 

53 schema.addField("flux_instFlux", type=np.float64) 

54 schema.addField("flux_instFluxErr", type=np.float64) 

55 schema.addField("flux_flag", type="Flag") 

56 self.table = afwTable.SourceTable.make(schema) 

57 self.table.definePsfFlux("flux") 

58 self.ss1 = afwTable.SourceCatalog(self.table) 

59 self.ss2 = afwTable.SourceCatalog(self.table) 

60 self.metadata = dafBase.PropertyList() 

61 

62 def tearDown(self): 

63 del self.table 

64 del self.metadata 

65 del self.ss1 

66 del self.ss2 

67 

68 def testIdentity(self): 

69 nobj = 1000 

70 for i in range(nobj): 

71 s = self.ss1.addNew() 

72 s.setId(i) 

73 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

74 (10 + 0.001*i) * lsst.geom.degrees) 

75 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

76 (10 + 0.001*i) * lsst.geom.degrees) 

77 

78 s = self.ss2.addNew() 

79 s.setId(2*nobj + i) 

80 # Give slight offsets for Coord testing of matches to/from catalog in checkMatchToFromCatalog() 

81 # Chosen such that the maximum offset (nobj*1E-7 deg = 0.36 arcsec) is within the maximum 

82 # distance (1 arcsec) in afwTable.matchRaDec. 

83 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

84 (10 + 0.0010001*i) * lsst.geom.degrees) 

85 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

86 (10 + 0.0010001*i) * lsst.geom.degrees) 

87 

88 mc = afwTable.MatchControl() 

89 mc.findOnlyClosest = False 

90 mat = afwTable.matchRaDec( 

91 self.ss1, self.ss2, 1.0*lsst.geom.arcseconds, mc) 

92 self.assertEqual(len(mat), nobj) 

93 

94 cat = afwTable.packMatches(mat) 

95 

96 mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2) 

97 

98 for m1, m2, c in zip(mat, mat2, cat): 

99 self.assertEqual(m1.first, m2.first) 

100 self.assertEqual(m1.second, m2.second) 

101 self.assertEqual(m1.distance, m2.distance) 

102 self.assertEqual(m1.first.getId(), c["first"]) 

103 self.assertEqual(m1.second.getId(), c["second"]) 

104 self.assertEqual(m1.distance, c["distance"]) 

105 

106 self.checkMatchToFromCatalog(mat, cat) 

107 

108 if False: 

109 s0 = mat[0][0] 

110 s1 = mat[0][1] 

111 print(s0.getRa(), s1.getRa(), s0.getId(), s1.getId()) 

112 

113 def testNaNPositions(self): 

114 ss1 = afwTable.SourceCatalog(self.table) 

115 ss2 = afwTable.SourceCatalog(self.table) 

116 for ss in (ss1, ss2): 

117 ss.addNew().set(afwTable.SourceTable.getCoordKey().getRa(), 

118 float('nan') * lsst.geom.radians) 

119 

120 ss.addNew().set(afwTable.SourceTable.getCoordKey().getDec(), 

121 float('nan') * lsst.geom.radians) 

122 

123 s = ss.addNew() 

124 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

125 0.0 * lsst.geom.radians) 

126 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

127 0.0 * lsst.geom.radians) 

128 

129 s = ss.addNew() 

130 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

131 float('nan') * lsst.geom.radians) 

132 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

133 float('nan') * lsst.geom.radians) 

134 

135 mc = afwTable.MatchControl() 

136 mc.findOnlyClosest = False 

137 mat = afwTable.matchRaDec(ss1, ss2, 1.0*lsst.geom.arcseconds, mc) 

138 self.assertEqual(len(mat), 1) 

139 

140 @unittest.skipIf(afwdataDir is None, "afwdata not setup") 

141 def testPhotometricCalib(self): 

142 """Test matching the CFHT catalogue (as generated using LSST code) to the SDSS catalogue 

143 """ 

144 

145 band = 2 # SDSS r 

146 

147 # Read SDSS catalogue 

148 with open(os.path.join(afwdataDir, "CFHT", "D2", "sdss.dat"), "r") as ifd: 

149 

150 sdss = afwTable.SourceCatalog(self.table) 

151 sdssSecondary = afwTable.SourceCatalog(self.table) 

152 

153 PRIMARY, SECONDARY = 1, 2 # values of mode 

154 

155 id = 0 

156 for line in ifd.readlines(): 

157 if re.search(r"^\s*#", line): 

158 continue 

159 

160 fields = line.split() 

161 objId = int(fields[0]) 

162 fields[1] 

163 mode = int(fields[2]) 

164 ra, dec = [float(f) for f in fields[3:5]] 

165 psfMags = [float(f) for f in fields[5:]] 

166 

167 if mode == PRIMARY: 

168 s = sdss.addNew() 

169 elif SECONDARY: 

170 s = sdssSecondary.addNew() 

171 

172 s.setId(objId) 

173 s.setRa(ra * lsst.geom.degrees) 

174 s.setDec(dec * lsst.geom.degrees) 

175 s.set(self.table.getPsfFluxSlot().getMeasKey(), psfMags[band]) 

176 

177 del ifd 

178 

179 # Read catalalogue built from the template image 

180 # Read SDSS catalogue 

181 with open(os.path.join(afwdataDir, "CFHT", "D2", "template.dat"), "r") as ifd: 

182 

183 template = afwTable.SourceCatalog(self.table) 

184 

185 id = 0 

186 for line in ifd.readlines(): 

187 if re.search(r"^\s*#", line): 

188 continue 

189 

190 fields = line.split() 

191 id, flags = [int(f) for f in fields[0:2]] 

192 ra, dec = [float(f) for f in fields[2:4]] 

193 flux = [float(f) for f in fields[4:]] 

194 

195 if flags & 0x1: # EDGE 

196 continue 

197 

198 s = template.addNew() 

199 s.setId(id) 

200 id += 1 

201 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

202 ra * lsst.geom.degrees) 

203 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

204 dec * lsst.geom.degrees) 

205 s.set(self.table.getPsfFluxSlot().getMeasKey(), flux[0]) 

206 

207 del ifd 

208 

209 # Actually do the match 

210 mc = afwTable.MatchControl() 

211 mc.findOnlyClosest = False 

212 

213 matches = afwTable.matchRaDec( 

214 sdss, template, 1.0*lsst.geom.arcseconds, mc) 

215 

216 self.assertEqual(len(matches), 901) 

217 

218 if False: 

219 for mat in matches: 

220 s0 = mat[0] 

221 s1 = mat[1] 

222 d = mat[2] 

223 print(s0.getRa(), s0.getDec(), s1.getRa(), 

224 s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux()) 

225 

226 # Actually do the match 

227 for s in sdssSecondary: 

228 sdss.append(s) 

229 

230 mc = afwTable.MatchControl() 

231 mc.symmetricMatch = False 

232 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds, mc) 

233 nmiss = 1 # one object doesn't match 

234 self.assertEqual(len(matches), len(sdssSecondary) - nmiss) 

235 

236 # Find the one that didn't match 

237 if False: 

238 matchIds = set() 

239 for s0, s1, d in matches: 

240 matchIds.add(s0.getId()) 

241 matchIds.add(s1.getId()) 

242 

243 for s in sdssSecondary: 

244 if s.getId() not in matchIds: 

245 print("RHL", s.getId()) 

246 

247 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds) 

248 self.assertEqual(len(matches), 2*(len(sdssSecondary) - nmiss)) 

249 

250 if False: 

251 for mat in matches: 

252 s0 = mat[0] 

253 s1 = mat[1] 

254 mat[2] 

255 print(s0.getId(), s1.getId(), s0.getRa(), s0.getDec(), end=' ') 

256 print(s1.getRa(), s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux()) 

257 

258 def testMismatches(self): 

259 """ Chech that matchRaDec works as expected when using 

260 the includeMismatches option 

261 """ 

262 cat1 = afwTable.SourceCatalog(self.table) 

263 cat2 = afwTable.SourceCatalog(self.table) 

264 nobj = 100 

265 for i in range(nobj): 

266 s1 = cat1.addNew() 

267 s2 = cat2.addNew() 

268 s1.setId(i) 

269 s2.setId(i) 

270 s1.set(afwTable.SourceTable.getCoordKey().getRa(), 

271 (10 + 0.0001*i) * lsst.geom.degrees) 

272 s2.set(afwTable.SourceTable.getCoordKey().getRa(), 

273 (10.005 + 0.0001*i) * lsst.geom.degrees) 

274 s1.set(afwTable.SourceTable.getCoordKey().getDec(), 

275 (10 + 0.0001*i) * lsst.geom.degrees) 

276 s2.set(afwTable.SourceTable.getCoordKey().getDec(), 

277 (10.005 + 0.0001*i) * lsst.geom.degrees) 

278 

279 for closest in (True, False): 

280 mc = afwTable.MatchControl() 

281 mc.findOnlyClosest = closest 

282 mc.includeMismatches = False 

283 matches = afwTable.matchRaDec( 

284 cat1, cat2, 1.0*lsst.geom.arcseconds, mc) 

285 mc.includeMismatches = True 

286 matchesMismatches = afwTable.matchRaDec( 

287 cat1, cat2, 1.0*lsst.geom.arcseconds, mc) 

288 

289 catMatches = afwTable.SourceCatalog(self.table) 

290 catMismatches = afwTable.SourceCatalog(self.table) 

291 for m in matchesMismatches: 

292 if m[1] is not None: 

293 if not any(x == m[0] for x in catMatches): 

294 catMatches.append(m[0]) 

295 else: 

296 catMismatches.append(m[0]) 

297 if closest: 

298 self.assertEqual(len(catMatches), len(matches)) 

299 matches2 = afwTable.matchRaDec( 

300 catMatches, cat2, 1.0*lsst.geom.arcseconds, mc) 

301 self.assertEqual(len(matches), len(matches2)) 

302 mc.includeMismatches = False 

303 noMatches = afwTable.matchRaDec( 

304 catMismatches, cat2, 1.0*lsst.geom.arcseconds, mc) 

305 self.assertEqual(len(noMatches), 0) 

306 

307 def checkMatchToFromCatalog(self, matches, catalog): 

308 """Check the conversion of matches to and from a catalog 

309 

310 Test the functions in lsst.afw.table.catalogMatches.py 

311 Note that the return types and schemas of these functions do not necessarily match 

312 those of the catalogs passed to them, so value entries are compared as opposed to 

313 comparing the attributes as a whole. 

314 """ 

315 catalog.setMetadata(self.metadata) 

316 matchMeta = catalog.getTable().getMetadata() 

317 matchToCat = afwTable.catalogMatches.matchesToCatalog( 

318 matches, matchMeta) 

319 matchFromCat = afwTable.catalogMatches.matchesFromCatalog(matchToCat) 

320 self.assertEqual(len(matches), len(matchToCat)) 

321 self.assertEqual(len(matches), len(matchFromCat)) 

322 

323 for mat, cat, catM, matchC in zip(matches, catalog, matchToCat, matchFromCat): 

324 self.assertEqual(mat.first.getId(), catM["ref_id"]) 

325 self.assertEqual(mat.first.getId(), matchC.first.getId()) 

326 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord()) 

327 self.assertEqual(mat.second.getId(), cat["second"]) 

328 self.assertEqual(mat.second.getId(), catM["src_id"]) 

329 self.assertEqual(mat.second.getId(), matchC.second.getId()) 

330 self.assertEqual((mat.first.getRa(), mat.first.getDec()), 

331 (catM["ref_coord_ra"], catM["ref_coord_dec"])) 

332 self.assertEqual((mat.second.getRa(), mat.second.getDec()), 

333 (catM["src_coord_ra"], catM["src_coord_dec"])) 

334 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord()) 

335 self.assertEqual(mat.second.getCoord(), matchC.second.getCoord()) 

336 self.assertEqual(mat.distance, matchC.distance) 

337 self.assertEqual(mat.distance, cat["distance"]) 

338 self.assertEqual(mat.distance, catM["distance"]) 

339 

340 def assertEqualFloat(self, value1, value2): 

341 """Compare floating point values, allowing for NAN 

342 """ 

343 self.assertTrue(value1 == value2 

344 or (np.isnan(value1) and np.isnan(value2))) 

345 

346 def testDistancePrecision(self): 

347 """Test for precision of the calculated distance 

348 

349 Check that the distance produced by matchRaDec is the same 

350 as the distance produced from calculating the separation 

351 between the matched coordinates. 

352 

353 Based on DM-13891. 

354 """ 

355 num = 1000 # Number of points 

356 radius = 0.5*lsst.geom.arcseconds # Matching radius 

357 tol = 1.0e-10 # Absolute tolerance 

358 rng = np.random.RandomState(12345) # I have the same combination on my luggage 

359 coordKey = afwTable.SourceTable.getCoordKey() 

360 raKey = coordKey.getRa() 

361 decKey = coordKey.getDec() 

362 for ii in range(num): 

363 src1 = self.ss1.addNew() 

364 src1.setId(ii) 

365 src1.set(raKey, (10 + 0.001*ii) * lsst.geom.degrees) 

366 src1.set(decKey, (10 + 0.001*ii) * lsst.geom.degrees) 

367 

368 src2 = self.ss2.addNew() 

369 src2.setId(2*num + ii) 

370 src2.set(coordKey, 

371 src1.getCoord().offset(rng.uniform(high=360)*lsst.geom.degrees, 

372 rng.uniform(high=radius.asArcseconds())*lsst.geom.arcseconds)) 

373 

374 matches = afwTable.matchRaDec(self.ss1, self.ss2, radius) 

375 dist1 = np.array([(mm.distance*lsst.geom.radians).asArcseconds() for mm in matches]) 

376 dist2 = np.array([mm.first.getCoord().separation(mm.second.getCoord()).asArcseconds() 

377 for mm in matches]) 

378 diff = dist1 - dist2 

379 self.assertLess(diff.std(), tol) # I get 4e-12 

380 self.assertFloatsAlmostEqual(dist1, dist2, atol=tol) 

381 

382 

383class TestMemory(lsst.utils.tests.MemoryTestCase): 

384 pass 

385 

386 

387def setup_module(module): 

388 lsst.utils.tests.init() 

389 

390 

391if __name__ == "__main__": 391 ↛ 392line 391 didn't jump to line 392, because the condition on line 391 was never true

392 lsst.utils.tests.init() 

393 unittest.main()