Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for matching SourceSets 

24 

25Run with: 

26 python test_sourceMatch.py 

27or 

28 pytest test_sourceMatch.py 

29""" 

30import os 

31import re 

32import unittest 

33 

34import numpy as np 

35 

36import lsst.geom 

37import lsst.afw.table as afwTable 

38import lsst.daf.base as dafBase 

39import lsst.utils.tests 

40import lsst.pex.exceptions as pexExcept 

41 

42try: 

43 afwdataDir = lsst.utils.getPackageDir("afwdata") 

44except pexExcept.NotFoundError: 

45 afwdataDir = None 

46 

47 

48class SourceMatchTestCase(lsst.utils.tests.TestCase): 

49 """A test case for matching SourceSets 

50 """ 

51 

52 def setUp(self): 

53 schema = afwTable.SourceTable.makeMinimalSchema() 

54 schema.addField("flux_instFlux", type=np.float64) 

55 schema.addField("flux_instFluxErr", type=np.float64) 

56 schema.addField("flux_flag", type="Flag") 

57 self.table = afwTable.SourceTable.make(schema) 

58 self.table.definePsfFlux("flux") 

59 self.ss1 = afwTable.SourceCatalog(self.table) 

60 self.ss2 = afwTable.SourceCatalog(self.table) 

61 self.metadata = dafBase.PropertyList() 

62 

63 def tearDown(self): 

64 del self.table 

65 del self.metadata 

66 del self.ss1 

67 del self.ss2 

68 

69 def testIdentity(self): 

70 nobj = 1000 

71 for i in range(nobj): 

72 s = self.ss1.addNew() 

73 s.setId(i) 

74 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

75 (10 + 0.001*i) * lsst.geom.degrees) 

76 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

77 (10 + 0.001*i) * lsst.geom.degrees) 

78 

79 s = self.ss2.addNew() 

80 s.setId(2*nobj + i) 

81 # Give slight offsets for Coord testing of matches to/from catalog in checkMatchToFromCatalog() 

82 # Chosen such that the maximum offset (nobj*1E-7 deg = 0.36 arcsec) is within the maximum 

83 # distance (1 arcsec) in afwTable.matchRaDec. 

84 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

85 (10 + 0.0010001*i) * lsst.geom.degrees) 

86 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

87 (10 + 0.0010001*i) * lsst.geom.degrees) 

88 

89 mc = afwTable.MatchControl() 

90 mc.findOnlyClosest = False 

91 mat = afwTable.matchRaDec( 

92 self.ss1, self.ss2, 1.0*lsst.geom.arcseconds, mc) 

93 self.assertEqual(len(mat), nobj) 

94 

95 cat = afwTable.packMatches(mat) 

96 

97 mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2) 

98 

99 for m1, m2, c in zip(mat, mat2, cat): 

100 self.assertEqual(m1.first, m2.first) 

101 self.assertEqual(m1.second, m2.second) 

102 self.assertEqual(m1.distance, m2.distance) 

103 self.assertEqual(m1.first.getId(), c["first"]) 

104 self.assertEqual(m1.second.getId(), c["second"]) 

105 self.assertEqual(m1.distance, c["distance"]) 

106 

107 self.checkMatchToFromCatalog(mat, cat) 

108 

109 if False: 

110 s0 = mat[0][0] 

111 s1 = mat[0][1] 

112 print(s0.getRa(), s1.getRa(), s0.getId(), s1.getId()) 

113 

114 def testNaNPositions(self): 

115 ss1 = afwTable.SourceCatalog(self.table) 

116 ss2 = afwTable.SourceCatalog(self.table) 

117 for ss in (ss1, ss2): 

118 ss.addNew().set(afwTable.SourceTable.getCoordKey().getRa(), 

119 float('nan') * lsst.geom.radians) 

120 

121 ss.addNew().set(afwTable.SourceTable.getCoordKey().getDec(), 

122 float('nan') * lsst.geom.radians) 

123 

124 s = ss.addNew() 

125 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

126 0.0 * lsst.geom.radians) 

127 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

128 0.0 * lsst.geom.radians) 

129 

130 s = ss.addNew() 

131 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

132 float('nan') * lsst.geom.radians) 

133 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

134 float('nan') * lsst.geom.radians) 

135 

136 mc = afwTable.MatchControl() 

137 mc.findOnlyClosest = False 

138 mat = afwTable.matchRaDec(ss1, ss2, 1.0*lsst.geom.arcseconds, mc) 

139 self.assertEqual(len(mat), 1) 

140 

141 @unittest.skipIf(afwdataDir is None, "afwdata not setup") 

142 def testPhotometricCalib(self): 

143 """Test matching the CFHT catalogue (as generated using LSST code) to the SDSS catalogue 

144 """ 

145 

146 band = 2 # SDSS r 

147 

148 # Read SDSS catalogue 

149 with open(os.path.join(afwdataDir, "CFHT", "D2", "sdss.dat"), "r") as ifd: 

150 

151 sdss = afwTable.SourceCatalog(self.table) 

152 sdssSecondary = afwTable.SourceCatalog(self.table) 

153 

154 PRIMARY, SECONDARY = 1, 2 # values of mode 

155 

156 id = 0 

157 for line in ifd.readlines(): 

158 if re.search(r"^\s*#", line): 

159 continue 

160 

161 fields = line.split() 

162 objId = int(fields[0]) 

163 fields[1] 

164 mode = int(fields[2]) 

165 ra, dec = [float(f) for f in fields[3:5]] 

166 psfMags = [float(f) for f in fields[5:]] 

167 

168 if mode == PRIMARY: 

169 s = sdss.addNew() 

170 elif SECONDARY: 

171 s = sdssSecondary.addNew() 

172 

173 s.setId(objId) 

174 s.setRa(ra * lsst.geom.degrees) 

175 s.setDec(dec * lsst.geom.degrees) 

176 s.set(self.table.getPsfFluxSlot().getMeasKey(), psfMags[band]) 

177 

178 del ifd 

179 

180 # Read catalalogue built from the template image 

181 # Read SDSS catalogue 

182 with open(os.path.join(afwdataDir, "CFHT", "D2", "template.dat"), "r") as ifd: 

183 

184 template = afwTable.SourceCatalog(self.table) 

185 

186 id = 0 

187 for line in ifd.readlines(): 

188 if re.search(r"^\s*#", line): 

189 continue 

190 

191 fields = line.split() 

192 id, flags = [int(f) for f in fields[0:2]] 

193 ra, dec = [float(f) for f in fields[2:4]] 

194 flux = [float(f) for f in fields[4:]] 

195 

196 if flags & 0x1: # EDGE 

197 continue 

198 

199 s = template.addNew() 

200 s.setId(id) 

201 id += 1 

202 s.set(afwTable.SourceTable.getCoordKey().getRa(), 

203 ra * lsst.geom.degrees) 

204 s.set(afwTable.SourceTable.getCoordKey().getDec(), 

205 dec * lsst.geom.degrees) 

206 s.set(self.table.getPsfFluxSlot().getMeasKey(), flux[0]) 

207 

208 del ifd 

209 

210 # Actually do the match 

211 mc = afwTable.MatchControl() 

212 mc.findOnlyClosest = False 

213 

214 matches = afwTable.matchRaDec( 

215 sdss, template, 1.0*lsst.geom.arcseconds, mc) 

216 

217 self.assertEqual(len(matches), 901) 

218 

219 if False: 

220 for mat in matches: 

221 s0 = mat[0] 

222 s1 = mat[1] 

223 d = mat[2] 

224 print(s0.getRa(), s0.getDec(), s1.getRa(), 

225 s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux()) 

226 

227 # Actually do the match 

228 for s in sdssSecondary: 

229 sdss.append(s) 

230 

231 mc = afwTable.MatchControl() 

232 mc.symmetricMatch = False 

233 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds, mc) 

234 nmiss = 1 # one object doesn't match 

235 self.assertEqual(len(matches), len(sdssSecondary) - nmiss) 

236 

237 # Find the one that didn't match 

238 if False: 

239 matchIds = set() 

240 for s0, s1, d in matches: 

241 matchIds.add(s0.getId()) 

242 matchIds.add(s1.getId()) 

243 

244 for s in sdssSecondary: 

245 if s.getId() not in matchIds: 

246 print("RHL", s.getId()) 

247 

248 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds) 

249 self.assertEqual(len(matches), 2*(len(sdssSecondary) - nmiss)) 

250 

251 if False: 

252 for mat in matches: 

253 s0 = mat[0] 

254 s1 = mat[1] 

255 mat[2] 

256 print(s0.getId(), s1.getId(), s0.getRa(), s0.getDec(), end=' ') 

257 print(s1.getRa(), s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux()) 

258 

259 def testMismatches(self): 

260 """ Chech that matchRaDec works as expected when using 

261 the includeMismatches option 

262 """ 

263 cat1 = afwTable.SourceCatalog(self.table) 

264 cat2 = afwTable.SourceCatalog(self.table) 

265 nobj = 100 

266 for i in range(nobj): 

267 s1 = cat1.addNew() 

268 s2 = cat2.addNew() 

269 s1.setId(i) 

270 s2.setId(i) 

271 s1.set(afwTable.SourceTable.getCoordKey().getRa(), 

272 (10 + 0.0001*i) * lsst.geom.degrees) 

273 s2.set(afwTable.SourceTable.getCoordKey().getRa(), 

274 (10.005 + 0.0001*i) * lsst.geom.degrees) 

275 s1.set(afwTable.SourceTable.getCoordKey().getDec(), 

276 (10 + 0.0001*i) * lsst.geom.degrees) 

277 s2.set(afwTable.SourceTable.getCoordKey().getDec(), 

278 (10.005 + 0.0001*i) * lsst.geom.degrees) 

279 

280 for closest in (True, False): 

281 mc = afwTable.MatchControl() 

282 mc.findOnlyClosest = closest 

283 mc.includeMismatches = False 

284 matches = afwTable.matchRaDec( 

285 cat1, cat2, 1.0*lsst.geom.arcseconds, mc) 

286 mc.includeMismatches = True 

287 matchesMismatches = afwTable.matchRaDec( 

288 cat1, cat2, 1.0*lsst.geom.arcseconds, mc) 

289 

290 catMatches = afwTable.SourceCatalog(self.table) 

291 catMismatches = afwTable.SourceCatalog(self.table) 

292 for m in matchesMismatches: 

293 if m[1] is not None: 

294 if not any(x == m[0] for x in catMatches): 

295 catMatches.append(m[0]) 

296 else: 

297 catMismatches.append(m[0]) 

298 if closest: 

299 self.assertEqual(len(catMatches), len(matches)) 

300 matches2 = afwTable.matchRaDec( 

301 catMatches, cat2, 1.0*lsst.geom.arcseconds, mc) 

302 self.assertEqual(len(matches), len(matches2)) 

303 mc.includeMismatches = False 

304 noMatches = afwTable.matchRaDec( 

305 catMismatches, cat2, 1.0*lsst.geom.arcseconds, mc) 

306 self.assertEqual(len(noMatches), 0) 

307 

308 def checkMatchToFromCatalog(self, matches, catalog): 

309 """Check the conversion of matches to and from a catalog 

310 

311 Test the functions in lsst.afw.table.catalogMatches.py 

312 Note that the return types and schemas of these functions do not necessarily match 

313 those of the catalogs passed to them, so value entries are compared as opposed to 

314 comparing the attributes as a whole. 

315 """ 

316 catalog.setMetadata(self.metadata) 

317 matchMeta = catalog.getTable().getMetadata() 

318 matchToCat = afwTable.catalogMatches.matchesToCatalog( 

319 matches, matchMeta) 

320 matchFromCat = afwTable.catalogMatches.matchesFromCatalog(matchToCat) 

321 self.assertEqual(len(matches), len(matchToCat)) 

322 self.assertEqual(len(matches), len(matchFromCat)) 

323 

324 for mat, cat, catM, matchC in zip(matches, catalog, matchToCat, matchFromCat): 

325 self.assertEqual(mat.first.getId(), catM["ref_id"]) 

326 self.assertEqual(mat.first.getId(), matchC.first.getId()) 

327 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord()) 

328 self.assertEqual(mat.second.getId(), cat["second"]) 

329 self.assertEqual(mat.second.getId(), catM["src_id"]) 

330 self.assertEqual(mat.second.getId(), matchC.second.getId()) 

331 self.assertEqual((mat.first.getRa(), mat.first.getDec()), 

332 (catM["ref_coord_ra"], catM["ref_coord_dec"])) 

333 self.assertEqual((mat.second.getRa(), mat.second.getDec()), 

334 (catM["src_coord_ra"], catM["src_coord_dec"])) 

335 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord()) 

336 self.assertEqual(mat.second.getCoord(), matchC.second.getCoord()) 

337 self.assertEqual(mat.distance, matchC.distance) 

338 self.assertEqual(mat.distance, cat["distance"]) 

339 self.assertEqual(mat.distance, catM["distance"]) 

340 

341 def assertEqualFloat(self, value1, value2): 

342 """Compare floating point values, allowing for NAN 

343 """ 

344 self.assertTrue(value1 == value2 

345 or (np.isnan(value1) and np.isnan(value2))) 

346 

347 def testDistancePrecision(self): 

348 """Test for precision of the calculated distance 

349 

350 Check that the distance produced by matchRaDec is the same 

351 as the distance produced from calculating the separation 

352 between the matched coordinates. 

353 

354 Based on DM-13891. 

355 """ 

356 num = 1000 # Number of points 

357 radius = 0.5*lsst.geom.arcseconds # Matching radius 

358 tol = 1.0e-10 # Absolute tolerance 

359 rng = np.random.RandomState(12345) # I have the same combination on my luggage 

360 coordKey = afwTable.SourceTable.getCoordKey() 

361 raKey = coordKey.getRa() 

362 decKey = coordKey.getDec() 

363 for ii in range(num): 

364 src1 = self.ss1.addNew() 

365 src1.setId(ii) 

366 src1.set(raKey, (10 + 0.001*ii) * lsst.geom.degrees) 

367 src1.set(decKey, (10 + 0.001*ii) * lsst.geom.degrees) 

368 

369 src2 = self.ss2.addNew() 

370 src2.setId(2*num + ii) 

371 src2.set(coordKey, 

372 src1.getCoord().offset(rng.uniform(high=360)*lsst.geom.degrees, 

373 rng.uniform(high=radius.asArcseconds())*lsst.geom.arcseconds)) 

374 

375 matches = afwTable.matchRaDec(self.ss1, self.ss2, radius) 

376 dist1 = np.array([(mm.distance*lsst.geom.radians).asArcseconds() for mm in matches]) 

377 dist2 = np.array([mm.first.getCoord().separation(mm.second.getCoord()).asArcseconds() 

378 for mm in matches]) 

379 diff = dist1 - dist2 

380 self.assertLess(diff.std(), tol) # I get 4e-12 

381 self.assertFloatsAlmostEqual(dist1, dist2, atol=tol) 

382 

383 

384class TestMemory(lsst.utils.tests.MemoryTestCase): 

385 pass 

386 

387 

388def setup_module(module): 

389 lsst.utils.tests.init() 

390 

391 

392if __name__ == "__main__": 392 ↛ 393line 392 didn't jump to line 393, because the condition on line 392 was never true

393 lsst.utils.tests.init() 

394 unittest.main()