Coverage for tests/test_sourceMatch.py: 10%
218 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-29 02:27 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-29 02:27 -0700
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Tests for matching SourceSets
25Run with:
26 python test_sourceMatch.py
27or
28 pytest test_sourceMatch.py
29"""
30import os
31import re
32import unittest
34import numpy as np
36import lsst.geom
37import lsst.afw.table as afwTable
38import lsst.daf.base as dafBase
39import lsst.utils.tests
41try:
42 afwdataDir = lsst.utils.getPackageDir("afwdata")
43except LookupError:
44 afwdataDir = None
47class SourceMatchTestCase(lsst.utils.tests.TestCase):
48 """A test case for matching SourceSets
49 """
51 def setUp(self):
52 schema = afwTable.SourceTable.makeMinimalSchema()
53 schema.addField("flux_instFlux", type=np.float64)
54 schema.addField("flux_instFluxErr", type=np.float64)
55 schema.addField("flux_flag", type="Flag")
56 self.table = afwTable.SourceTable.make(schema)
57 self.table.definePsfFlux("flux")
58 self.ss1 = afwTable.SourceCatalog(self.table)
59 self.ss2 = afwTable.SourceCatalog(self.table)
60 self.metadata = dafBase.PropertyList()
62 def tearDown(self):
63 del self.table
64 del self.metadata
65 del self.ss1
66 del self.ss2
68 def testIdentity(self):
69 nobj = 1000
70 for i in range(nobj):
71 s = self.ss1.addNew()
72 s.setId(i)
73 s.set(afwTable.SourceTable.getCoordKey().getRa(),
74 (10 + 0.001*i) * lsst.geom.degrees)
75 s.set(afwTable.SourceTable.getCoordKey().getDec(),
76 (10 + 0.001*i) * lsst.geom.degrees)
78 s = self.ss2.addNew()
79 s.setId(2*nobj + i)
80 # Give slight offsets for Coord testing of matches to/from catalog in checkMatchToFromCatalog()
81 # Chosen such that the maximum offset (nobj*1E-7 deg = 0.36 arcsec) is within the maximum
82 # distance (1 arcsec) in afwTable.matchRaDec.
83 s.set(afwTable.SourceTable.getCoordKey().getRa(),
84 (10 + 0.0010001*i) * lsst.geom.degrees)
85 s.set(afwTable.SourceTable.getCoordKey().getDec(),
86 (10 + 0.0010001*i) * lsst.geom.degrees)
88 mc = afwTable.MatchControl()
89 mc.findOnlyClosest = False
90 mat = afwTable.matchRaDec(
91 self.ss1, self.ss2, 1.0*lsst.geom.arcseconds, mc)
92 self.assertEqual(len(mat), nobj)
94 cat = afwTable.packMatches(mat)
96 mat2 = afwTable.unpackMatches(cat, self.ss1, self.ss2)
98 for m1, m2, c in zip(mat, mat2, cat):
99 self.assertEqual(m1.first, m2.first)
100 self.assertEqual(m1.second, m2.second)
101 self.assertEqual(m1.distance, m2.distance)
102 self.assertEqual(m1.first.getId(), c["first"])
103 self.assertEqual(m1.second.getId(), c["second"])
104 self.assertEqual(m1.distance, c["distance"])
106 self.checkMatchToFromCatalog(mat, cat)
108 if False:
109 s0 = mat[0][0]
110 s1 = mat[0][1]
111 print(s0.getRa(), s1.getRa(), s0.getId(), s1.getId())
113 def testNaNPositions(self):
114 ss1 = afwTable.SourceCatalog(self.table)
115 ss2 = afwTable.SourceCatalog(self.table)
116 for ss in (ss1, ss2):
117 ss.addNew().set(afwTable.SourceTable.getCoordKey().getRa(),
118 float('nan') * lsst.geom.radians)
120 ss.addNew().set(afwTable.SourceTable.getCoordKey().getDec(),
121 float('nan') * lsst.geom.radians)
123 s = ss.addNew()
124 s.set(afwTable.SourceTable.getCoordKey().getRa(),
125 0.0 * lsst.geom.radians)
126 s.set(afwTable.SourceTable.getCoordKey().getDec(),
127 0.0 * lsst.geom.radians)
129 s = ss.addNew()
130 s.set(afwTable.SourceTable.getCoordKey().getRa(),
131 float('nan') * lsst.geom.radians)
132 s.set(afwTable.SourceTable.getCoordKey().getDec(),
133 float('nan') * lsst.geom.radians)
135 mc = afwTable.MatchControl()
136 mc.findOnlyClosest = False
137 mat = afwTable.matchRaDec(ss1, ss2, 1.0*lsst.geom.arcseconds, mc)
138 self.assertEqual(len(mat), 1)
140 @unittest.skipIf(afwdataDir is None, "afwdata not setup")
141 def testPhotometricCalib(self):
142 """Test matching the CFHT catalogue (as generated using LSST code) to the SDSS catalogue
143 """
145 band = 2 # SDSS r
147 # Read SDSS catalogue
148 with open(os.path.join(afwdataDir, "CFHT", "D2", "sdss.dat"), "r") as ifd:
150 sdss = afwTable.SourceCatalog(self.table)
151 sdssSecondary = afwTable.SourceCatalog(self.table)
153 PRIMARY, SECONDARY = 1, 2 # values of mode
155 id = 0
156 for line in ifd.readlines():
157 if re.search(r"^\s*#", line):
158 continue
160 fields = line.split()
161 objId = int(fields[0])
162 fields[1]
163 mode = int(fields[2])
164 ra, dec = [float(f) for f in fields[3:5]]
165 psfMags = [float(f) for f in fields[5:]]
167 if mode == PRIMARY:
168 s = sdss.addNew()
169 elif SECONDARY:
170 s = sdssSecondary.addNew()
172 s.setId(objId)
173 s.setRa(ra * lsst.geom.degrees)
174 s.setDec(dec * lsst.geom.degrees)
175 s.set(self.table.getPsfFluxSlot().getMeasKey(), psfMags[band])
177 del ifd
179 # Read catalalogue built from the template image
180 # Read SDSS catalogue
181 with open(os.path.join(afwdataDir, "CFHT", "D2", "template.dat"), "r") as ifd:
183 template = afwTable.SourceCatalog(self.table)
185 id = 0
186 for line in ifd.readlines():
187 if re.search(r"^\s*#", line):
188 continue
190 fields = line.split()
191 id, flags = [int(f) for f in fields[0:2]]
192 ra, dec = [float(f) for f in fields[2:4]]
193 flux = [float(f) for f in fields[4:]]
195 if flags & 0x1: # EDGE
196 continue
198 s = template.addNew()
199 s.setId(id)
200 id += 1
201 s.set(afwTable.SourceTable.getCoordKey().getRa(),
202 ra * lsst.geom.degrees)
203 s.set(afwTable.SourceTable.getCoordKey().getDec(),
204 dec * lsst.geom.degrees)
205 s.set(self.table.getPsfFluxSlot().getMeasKey(), flux[0])
207 del ifd
209 # Actually do the match
210 mc = afwTable.MatchControl()
211 mc.findOnlyClosest = False
213 matches = afwTable.matchRaDec(
214 sdss, template, 1.0*lsst.geom.arcseconds, mc)
216 self.assertEqual(len(matches), 901)
218 if False:
219 for mat in matches:
220 s0 = mat[0]
221 s1 = mat[1]
222 d = mat[2]
223 print(s0.getRa(), s0.getDec(), s1.getRa(),
224 s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux())
226 # Actually do the match
227 for s in sdssSecondary:
228 sdss.append(s)
230 mc = afwTable.MatchControl()
231 mc.symmetricMatch = False
232 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds, mc)
233 nmiss = 1 # one object doesn't match
234 self.assertEqual(len(matches), len(sdssSecondary) - nmiss)
236 # Find the one that didn't match
237 if False:
238 matchIds = set()
239 for s0, s1, d in matches:
240 matchIds.add(s0.getId())
241 matchIds.add(s1.getId())
243 for s in sdssSecondary:
244 if s.getId() not in matchIds:
245 print("RHL", s.getId())
247 matches = afwTable.matchRaDec(sdss, 1.0*lsst.geom.arcseconds)
248 self.assertEqual(len(matches), 2*(len(sdssSecondary) - nmiss))
250 if False:
251 for mat in matches:
252 s0 = mat[0]
253 s1 = mat[1]
254 mat[2]
255 print(s0.getId(), s1.getId(), s0.getRa(), s0.getDec(), end=' ')
256 print(s1.getRa(), s1.getDec(), s0.getPsfInstFlux(), s1.getPsfInstFlux())
258 def testMismatches(self):
259 """ Chech that matchRaDec works as expected when using
260 the includeMismatches option
261 """
262 cat1 = afwTable.SourceCatalog(self.table)
263 cat2 = afwTable.SourceCatalog(self.table)
264 nobj = 100
265 for i in range(nobj):
266 s1 = cat1.addNew()
267 s2 = cat2.addNew()
268 s1.setId(i)
269 s2.setId(i)
270 s1.set(afwTable.SourceTable.getCoordKey().getRa(),
271 (10 + 0.0001*i) * lsst.geom.degrees)
272 s2.set(afwTable.SourceTable.getCoordKey().getRa(),
273 (10.005 + 0.0001*i) * lsst.geom.degrees)
274 s1.set(afwTable.SourceTable.getCoordKey().getDec(),
275 (10 + 0.0001*i) * lsst.geom.degrees)
276 s2.set(afwTable.SourceTable.getCoordKey().getDec(),
277 (10.005 + 0.0001*i) * lsst.geom.degrees)
279 for closest in (True, False):
280 mc = afwTable.MatchControl()
281 mc.findOnlyClosest = closest
282 mc.includeMismatches = False
283 matches = afwTable.matchRaDec(
284 cat1, cat2, 1.0*lsst.geom.arcseconds, mc)
285 mc.includeMismatches = True
286 matchesMismatches = afwTable.matchRaDec(
287 cat1, cat2, 1.0*lsst.geom.arcseconds, mc)
289 catMatches = afwTable.SourceCatalog(self.table)
290 catMismatches = afwTable.SourceCatalog(self.table)
291 for m in matchesMismatches:
292 if m[1] is not None:
293 if not any(x == m[0] for x in catMatches):
294 catMatches.append(m[0])
295 else:
296 catMismatches.append(m[0])
297 if closest:
298 self.assertEqual(len(catMatches), len(matches))
299 matches2 = afwTable.matchRaDec(
300 catMatches, cat2, 1.0*lsst.geom.arcseconds, mc)
301 self.assertEqual(len(matches), len(matches2))
302 mc.includeMismatches = False
303 noMatches = afwTable.matchRaDec(
304 catMismatches, cat2, 1.0*lsst.geom.arcseconds, mc)
305 self.assertEqual(len(noMatches), 0)
307 def checkMatchToFromCatalog(self, matches, catalog):
308 """Check the conversion of matches to and from a catalog
310 Test the functions in lsst.afw.table.catalogMatches.py
311 Note that the return types and schemas of these functions do not necessarily match
312 those of the catalogs passed to them, so value entries are compared as opposed to
313 comparing the attributes as a whole.
314 """
315 catalog.setMetadata(self.metadata)
316 matchMeta = catalog.getTable().getMetadata()
317 matchToCat = afwTable.catalogMatches.matchesToCatalog(
318 matches, matchMeta)
319 matchFromCat = afwTable.catalogMatches.matchesFromCatalog(matchToCat)
320 self.assertEqual(len(matches), len(matchToCat))
321 self.assertEqual(len(matches), len(matchFromCat))
323 for mat, cat, catM, matchC in zip(matches, catalog, matchToCat, matchFromCat):
324 self.assertEqual(mat.first.getId(), catM["ref_id"])
325 self.assertEqual(mat.first.getId(), matchC.first.getId())
326 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord())
327 self.assertEqual(mat.second.getId(), cat["second"])
328 self.assertEqual(mat.second.getId(), catM["src_id"])
329 self.assertEqual(mat.second.getId(), matchC.second.getId())
330 self.assertEqual((mat.first.getRa(), mat.first.getDec()),
331 (catM["ref_coord_ra"], catM["ref_coord_dec"]))
332 self.assertEqual((mat.second.getRa(), mat.second.getDec()),
333 (catM["src_coord_ra"], catM["src_coord_dec"]))
334 self.assertEqual(mat.first.getCoord(), matchC.first.getCoord())
335 self.assertEqual(mat.second.getCoord(), matchC.second.getCoord())
336 self.assertEqual(mat.distance, matchC.distance)
337 self.assertEqual(mat.distance, cat["distance"])
338 self.assertEqual(mat.distance, catM["distance"])
340 def assertEqualFloat(self, value1, value2):
341 """Compare floating point values, allowing for NAN
342 """
343 self.assertTrue(value1 == value2
344 or (np.isnan(value1) and np.isnan(value2)))
346 def testDistancePrecision(self):
347 """Test for precision of the calculated distance
349 Check that the distance produced by matchRaDec is the same
350 as the distance produced from calculating the separation
351 between the matched coordinates.
353 Based on DM-13891.
354 """
355 num = 1000 # Number of points
356 radius = 0.5*lsst.geom.arcseconds # Matching radius
357 tol = 1.0e-10 # Absolute tolerance
358 rng = np.random.RandomState(12345) # I have the same combination on my luggage
359 coordKey = afwTable.SourceTable.getCoordKey()
360 raKey = coordKey.getRa()
361 decKey = coordKey.getDec()
362 for ii in range(num):
363 src1 = self.ss1.addNew()
364 src1.setId(ii)
365 src1.set(raKey, (10 + 0.001*ii) * lsst.geom.degrees)
366 src1.set(decKey, (10 + 0.001*ii) * lsst.geom.degrees)
368 src2 = self.ss2.addNew()
369 src2.setId(2*num + ii)
370 src2.set(coordKey,
371 src1.getCoord().offset(rng.uniform(high=360)*lsst.geom.degrees,
372 rng.uniform(high=radius.asArcseconds())*lsst.geom.arcseconds))
374 matches = afwTable.matchRaDec(self.ss1, self.ss2, radius)
375 dist1 = np.array([(mm.distance*lsst.geom.radians).asArcseconds() for mm in matches])
376 dist2 = np.array([mm.first.getCoord().separation(mm.second.getCoord()).asArcseconds()
377 for mm in matches])
378 diff = dist1 - dist2
379 self.assertLess(diff.std(), tol) # I get 4e-12
380 self.assertFloatsAlmostEqual(dist1, dist2, atol=tol)
383class TestMemory(lsst.utils.tests.MemoryTestCase):
384 pass
387def setup_module(module):
388 lsst.utils.tests.init()
391if __name__ == "__main__": 391 ↛ 392line 391 didn't jump to line 392, because the condition on line 391 was never true
392 lsst.utils.tests.init()
393 unittest.main()