Coverage for tests/test_sourceMatch.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Tests for matching SourceSets
25Run with:
26 python test_sourceMatch.py
27or
28 pytest test_sourceMatch.py
29"""
30import os
31import re
32import unittest
34import numpy as np
36import lsst.geom
37import lsst.afw.table as afwTable
38import lsst.daf.base as dafBase
39import lsst.utils.tests
40import lsst.pex.exceptions as pexExcept
42try:
43 afwdataDir = lsst.utils.getPackageDir("afwdata")
44except pexExcept.NotFoundError:
45 afwdataDir = None
48class SourceMatchTestCase(lsst.utils.tests.TestCase):
49 """A test case for matching SourceSets
50 """
52 def setUp(self):
53 schema = afwTable.SourceTable.makeMinimalSchema()
54 schema.addField("flux_instFlux", type=np.float64)
55 schema.addField("flux_instFluxErr", type=np.float64)
56 schema.addField("flux_flag", type="Flag")
57 self.table = afwTable.SourceTable.make(schema)
58 self.table.definePsfFlux("flux")
59 self.ss1 = afwTable.SourceCatalog(self.table)
60 self.ss2 = afwTable.SourceCatalog(self.table)
61 self.metadata = dafBase.PropertyList()
63 def tearDown(self):
64 del self.table
65 del self.metadata
66 del self.ss1
67 del self.ss2
69 def testIdentity(self):
70 nobj = 1000
71 for i in range(nobj):
72 s = self.ss1.addNew()
73 s.setId(i)
74 s.set(afwTable.SourceTable.getCoordKey().getRa(),
75 (10 + 0.001*i) * lsst.geom.degrees)
76 s.set(afwTable.SourceTable.getCoordKey().getDec(),
77 (10 + 0.001*i) * lsst.geom.degrees)
79 s = self.ss2.addNew()
80 s.setId(2*nobj + i)
81 # Give slight offsets for Coord testing of matches to/from catalog in checkMatchToFromCatalog()
82 # Chosen such that the maximum offset (nobj*1E-7 deg = 0.36 arcsec) is within the maximum
83 # distance (1 arcsec) in afwTable.matchRaDec.
84 s.set(afwTable.SourceTable.getCoordKey().getRa(),
85 (10 + 0.0010001*i) * lsst.geom.degrees)
86 s.set(afwTable.SourceTable.getCoordKey().getDec(),
87 (10 + 0.0010001*i) * lsst.geom.degrees)
89 mc = afwTable.MatchControl()
90 mc.findOnlyClosest = False
91 mat = afwTable.matchRaDec(
92 self.ss1, self.ss2, 1.0*lsst.geom.arcseconds, mc)
93 self.assertEqual(len(mat), nobj)
95 cat = afwTable.packMatches(mat)
97 mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2)
99 for m1, m2, c in zip(mat, mat2, cat):
100 self.assertEqual(m1.first, m2.first)
101 self.assertEqual(m1.second, m2.second)
102 self.assertEqual(m1.distance, m2.distance)
103 self.assertEqual(m1.first.getId(), c["first"])
104 self.assertEqual(m1.second.getId(), c["second"])
105 self.assertEqual(m1.distance, c["distance"])
107 self.checkMatchToFromCatalog(mat, cat)
109 if False:
110 s0 = mat[0][0]
111 s1 = mat[0][1]
112 print(s0.getRa(), s1.getRa(), s0.getId(), s1.getId())
114 def testNaNPositions(self):
115 ss1 = afwTable.SourceCatalog(self.table)
116 ss2 = afwTable.SourceCatalog(self.table)
117 for ss in (ss1, ss2):
118 ss.addNew().set(afwTable.SourceTable.getCoordKey().getRa(),
119 float('nan') * lsst.geom.radians)
121 ss.addNew().set(afwTable.SourceTable.getCoordKey().getDec(),
122 float('nan') * lsst.geom.radians)
124 s = ss.addNew()
125 s.set(afwTable.SourceTable.getCoordKey().getRa(),
126 0.0 * lsst.geom.radians)
127 s.set(afwTable.SourceTable.getCoordKey().getDec(),
128 0.0 * lsst.geom.radians)
130 s = ss.addNew()
131 s.set(afwTable.SourceTable.getCoordKey().getRa(),
132 float('nan') * lsst.geom.radians)
133 s.set(afwTable.SourceTable.getCoordKey().getDec(),
134 float('nan') * lsst.geom.radians)
136 mc = afwTable.MatchControl()
137 mc.findOnlyClosest = False
138 mat = afwTable.matchRaDec(ss1, ss2, 1.0*lsst.geom.arcseconds, mc)
139 self.assertEqual(len(mat), 1)
141 @unittest.skipIf(afwdataDir is None, "afwdata not setup")
142 def testPhotometricCalib(self):
143 """Test matching the CFHT catalogue (as generated using LSST code) to the SDSS catalogue
144 """
146 band = 2 # SDSS r
148 # Read SDSS catalogue
149 with open(os.path.join(afwdataDir, "CFHT", "D2", "sdss.dat"), "r") as ifd:
151 sdss = afwTable.SourceCatalog(self.table)
152 sdssSecondary = afwTable.SourceCatalog(self.table)
154 PRIMARY, SECONDARY = 1, 2 # values of mode
156 id = 0
157 for line in ifd.readlines():
158 if re.search(r"^\s*#", line):
159 continue
161 fields = line.split()
162 objId = int(fields[0])
163 fields[1]
164 mode = int(fields[2])
165 ra, dec = [float(f) for f in fields[3:5]]
166 psfMags = [float(f) for f in fields[5:]]
168 if mode == PRIMARY:
169 s = sdss.addNew()
170 elif SECONDARY:
171 s = sdssSecondary.addNew()
173 s.setId(objId)
174 s.setRa(ra * lsst.geom.degrees)
175 s.setDec(dec * lsst.geom.degrees)
176 s.set(self.table.getPsfFluxSlot().getMeasKey(), psfMags[band])
178 del ifd
180 # Read catalalogue built from the template image
181 # Read SDSS catalogue
182 with open(os.path.join(afwdataDir, "CFHT", "D2", "template.dat"), "r") as ifd:
184 template = afwTable.SourceCatalog(self.table)
186 id = 0
187 for line in ifd.readlines():
188 if re.search(r"^\s*#", line):
189 continue
191 fields = line.split()
192 id, flags = [int(f) for f in fields[0:2]]
193 ra, dec = [float(f) for f in fields[2:4]]
194 flux = [float(f) for f in fields[4:]]
196 if flags & 0x1: # EDGE
197 continue
199 s = template.addNew()
200 s.setId(id)
201 id += 1
202 s.set(afwTable.SourceTable.getCoordKey().getRa(),
203 ra * lsst.geom.degrees)
204 s.set(afwTable.SourceTable.getCoordKey().getDec(),
205 dec * lsst.geom.degrees)
206 s.set(self.table.getPsfFluxSlot().getMeasKey(), flux[0])
208 del ifd
210 # Actually do the match
211 mc = afwTable.MatchControl()
212 mc.findOnlyClosest = False
214 matches = afwTable.matchRaDec(
215 sdss, template, 1.0*lsst.geom.arcseconds, mc)
217 self.assertEqual(len(matches), 901)
219 if False:
220 for mat in matches:
221 s0 = mat[0]
222 s1 = mat[1]
223 d = mat[2]
224 print(s0.getRa(), s0.getDec(), s1.getRa(),
225 s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux())
227 # Actually do the match
228 for s in sdssSecondary:
229 sdss.append(s)
231 mc = afwTable.MatchControl()
232 mc.symmetricMatch = False
233 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds, mc)
234 nmiss = 1 # one object doesn't match
235 self.assertEqual(len(matches), len(sdssSecondary) - nmiss)
237 # Find the one that didn't match
238 if False:
239 matchIds = set()
240 for s0, s1, d in matches:
241 matchIds.add(s0.getId())
242 matchIds.add(s1.getId())
244 for s in sdssSecondary:
245 if s.getId() not in matchIds:
246 print("RHL", s.getId())
248 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds)
249 self.assertEqual(len(matches), 2*(len(sdssSecondary) - nmiss))
251 if False:
252 for mat in matches:
253 s0 = mat[0]
254 s1 = mat[1]
255 mat[2]
256 print(s0.getId(), s1.getId(), s0.getRa(), s0.getDec(), end=' ')
257 print(s1.getRa(), s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux())
259 def testMismatches(self):
260 """ Chech that matchRaDec works as expected when using
261 the includeMismatches option
262 """
263 cat1 = afwTable.SourceCatalog(self.table)
264 cat2 = afwTable.SourceCatalog(self.table)
265 nobj = 100
266 for i in range(nobj):
267 s1 = cat1.addNew()
268 s2 = cat2.addNew()
269 s1.setId(i)
270 s2.setId(i)
271 s1.set(afwTable.SourceTable.getCoordKey().getRa(),
272 (10 + 0.0001*i) * lsst.geom.degrees)
273 s2.set(afwTable.SourceTable.getCoordKey().getRa(),
274 (10.005 + 0.0001*i) * lsst.geom.degrees)
275 s1.set(afwTable.SourceTable.getCoordKey().getDec(),
276 (10 + 0.0001*i) * lsst.geom.degrees)
277 s2.set(afwTable.SourceTable.getCoordKey().getDec(),
278 (10.005 + 0.0001*i) * lsst.geom.degrees)
280 for closest in (True, False):
281 mc = afwTable.MatchControl()
282 mc.findOnlyClosest = closest
283 mc.includeMismatches = False
284 matches = afwTable.matchRaDec(
285 cat1, cat2, 1.0*lsst.geom.arcseconds, mc)
286 mc.includeMismatches = True
287 matchesMismatches = afwTable.matchRaDec(
288 cat1, cat2, 1.0*lsst.geom.arcseconds, mc)
290 catMatches = afwTable.SourceCatalog(self.table)
291 catMismatches = afwTable.SourceCatalog(self.table)
292 for m in matchesMismatches:
293 if m[1] is not None:
294 if not any(x == m[0] for x in catMatches):
295 catMatches.append(m[0])
296 else:
297 catMismatches.append(m[0])
298 if closest:
299 self.assertEqual(len(catMatches), len(matches))
300 matches2 = afwTable.matchRaDec(
301 catMatches, cat2, 1.0*lsst.geom.arcseconds, mc)
302 self.assertEqual(len(matches), len(matches2))
303 mc.includeMismatches = False
304 noMatches = afwTable.matchRaDec(
305 catMismatches, cat2, 1.0*lsst.geom.arcseconds, mc)
306 self.assertEqual(len(noMatches), 0)
308 def checkMatchToFromCatalog(self, matches, catalog):
309 """Check the conversion of matches to and from a catalog
311 Test the functions in lsst.afw.table.catalogMatches.py
312 Note that the return types and schemas of these functions do not necessarily match
313 those of the catalogs passed to them, so value entries are compared as opposed to
314 comparing the attributes as a whole.
315 """
316 catalog.setMetadata(self.metadata)
317 matchMeta = catalog.getTable().getMetadata()
318 matchToCat = afwTable.catalogMatches.matchesToCatalog(
319 matches, matchMeta)
320 matchFromCat = afwTable.catalogMatches.matchesFromCatalog(matchToCat)
321 self.assertEqual(len(matches), len(matchToCat))
322 self.assertEqual(len(matches), len(matchFromCat))
324 for mat, cat, catM, matchC in zip(matches, catalog, matchToCat, matchFromCat):
325 self.assertEqual(mat.first.getId(), catM["ref_id"])
326 self.assertEqual(mat.first.getId(), matchC.first.getId())
327 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord())
328 self.assertEqual(mat.second.getId(), cat["second"])
329 self.assertEqual(mat.second.getId(), catM["src_id"])
330 self.assertEqual(mat.second.getId(), matchC.second.getId())
331 self.assertEqual((mat.first.getRa(), mat.first.getDec()),
332 (catM["ref_coord_ra"], catM["ref_coord_dec"]))
333 self.assertEqual((mat.second.getRa(), mat.second.getDec()),
334 (catM["src_coord_ra"], catM["src_coord_dec"]))
335 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord())
336 self.assertEqual(mat.second.getCoord(), matchC.second.getCoord())
337 self.assertEqual(mat.distance, matchC.distance)
338 self.assertEqual(mat.distance, cat["distance"])
339 self.assertEqual(mat.distance, catM["distance"])
341 def assertEqualFloat(self, value1, value2):
342 """Compare floating point values, allowing for NAN
343 """
344 self.assertTrue(value1 == value2
345 or (np.isnan(value1) and np.isnan(value2)))
347 def testDistancePrecision(self):
348 """Test for precision of the calculated distance
350 Check that the distance produced by matchRaDec is the same
351 as the distance produced from calculating the separation
352 between the matched coordinates.
354 Based on DM-13891.
355 """
356 num = 1000 # Number of points
357 radius = 0.5*lsst.geom.arcseconds # Matching radius
358 tol = 1.0e-10 # Absolute tolerance
359 rng = np.random.RandomState(12345) # I have the same combination on my luggage
360 coordKey = afwTable.SourceTable.getCoordKey()
361 raKey = coordKey.getRa()
362 decKey = coordKey.getDec()
363 for ii in range(num):
364 src1 = self.ss1.addNew()
365 src1.setId(ii)
366 src1.set(raKey, (10 + 0.001*ii) * lsst.geom.degrees)
367 src1.set(decKey, (10 + 0.001*ii) * lsst.geom.degrees)
369 src2 = self.ss2.addNew()
370 src2.setId(2*num + ii)
371 src2.set(coordKey,
372 src1.getCoord().offset(rng.uniform(high=360)*lsst.geom.degrees,
373 rng.uniform(high=radius.asArcseconds())*lsst.geom.arcseconds))
375 matches = afwTable.matchRaDec(self.ss1, self.ss2, radius)
376 dist1 = np.array([(mm.distance*lsst.geom.radians).asArcseconds() for mm in matches])
377 dist2 = np.array([mm.first.getCoord().separation(mm.second.getCoord()).asArcseconds()
378 for mm in matches])
379 diff = dist1 - dist2
380 self.assertLess(diff.std(), tol) # I get 4e-12
381 self.assertFloatsAlmostEqual(dist1, dist2, atol=tol)
384class TestMemory(lsst.utils.tests.MemoryTestCase):
385 pass
388def setup_module(module):
389 lsst.utils.tests.init()
392if __name__ == "__main__": 392 ↛ 393line 392 didn't jump to line 393, because the condition on line 392 was never true
393 lsst.utils.tests.init()
394 unittest.main()