Coverage for tests/test_sourceTable.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Tests for table.SourceTable
25Run with:
26 python test_sourceTable.py
27or
28 pytest test_sourceTable.py
29"""
30import os
31import unittest
32import tempfile
33import pickle
34import math
36import numpy as np
38import lsst.utils.tests
39import lsst.pex.exceptions
40import lsst.geom
41import lsst.afw.table
42import lsst.afw.geom
43import lsst.afw.image
44import lsst.afw.detection
46testPath = os.path.abspath(os.path.dirname(__file__))
49def makeArray(size, dtype):
50 return np.array(np.random.randn(*size), dtype=dtype)
53def makeCov(size, dtype):
54 m = np.array(np.random.randn(size, size), dtype=dtype)
55 return np.dot(m, m.transpose())
58def makeWcs():
59 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees,
60 5.090329*lsst.geom.degrees)
61 crpix = lsst.geom.Point2D(2036.0, 2000.0)
62 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5])
63 cdMatrix.shape = (2, 2)
64 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
67class SourceTableTestCase(lsst.utils.tests.TestCase):
69 def fillRecord(self, record):
70 record.set(self.instFluxKey, np.random.randn())
71 record.set(self.instFluxErrKey, np.random.randn())
72 record.set(self.centroidKey.getX(), np.random.randn())
73 record.set(self.centroidKey.getY(), np.random.randn())
74 record.set(self.xErrKey, np.random.randn())
75 record.set(self.yErrKey, np.random.randn())
76 record.set(self.shapeKey.getIxx(), np.random.randn())
77 record.set(self.shapeKey.getIyy(), np.random.randn())
78 record.set(self.shapeKey.getIxy(), np.random.randn())
79 record.set(self.xxErrKey, np.random.randn())
80 record.set(self.yyErrKey, np.random.randn())
81 record.set(self.xyErrKey, np.random.randn())
82 record.set(self.fluxFlagKey, np.random.randn() > 0)
83 record.set(self.centroidFlagKey, np.random.randn() > 0)
84 record.set(self.shapeFlagKey, np.random.randn() > 0)
86 def setUp(self):
87 np.random.seed(1)
88 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema()
89 self.instFluxKey = self.schema.addField("a_instFlux", type="D")
90 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D")
91 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag")
93 # the meas field is added using a functor key, but the error is added
94 # as scalars, as we lack a ResultKey functor as exists in meas_base
95 self.centroidKey = lsst.afw.table.Point2DKey.addFields(
96 self.schema, "b", "", "pixel")
97 self.xErrKey = self.schema.addField("b_xErr", type="F")
98 self.yErrKey = self.schema.addField("b_yErr", type="F")
99 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag")
101 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields(
102 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL)
103 self.xxErrKey = self.schema.addField("c_xxErr", type="F")
104 self.xyErrKey = self.schema.addField("c_xyErr", type="F")
105 self.yyErrKey = self.schema.addField("c_yyErr", type="F")
106 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag")
108 self.table = lsst.afw.table.SourceTable.make(self.schema)
109 self.catalog = lsst.afw.table.SourceCatalog(self.table)
110 self.record = self.catalog.addNew()
111 self.fillRecord(self.record)
112 self.record.setId(50)
113 self.fillRecord(self.catalog.addNew())
114 self.fillRecord(self.catalog.addNew())
116 def tearDown(self):
117 del self.schema
118 del self.record
119 del self.table
120 del self.catalog
122 def checkCanonical(self):
123 self.assertEqual(self.record.get(self.instFluxKey),
124 self.record.getPsfInstFlux())
125 self.assertEqual(self.record.get(self.fluxFlagKey),
126 self.record.getPsfFluxFlag())
127 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b")
128 self.assertEqual(self.centroidKey.get(self.record),
129 self.record.getCentroid())
130 self.assertFloatsAlmostEqual(
131 math.fabs(self.record.get(self.xErrKey)),
132 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6)
133 self.assertFloatsAlmostEqual(
134 math.fabs(self.record.get(self.yErrKey)),
135 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6)
136 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c")
137 self.assertEqual(self.shapeKey.get(self.record),
138 self.record.getShape())
139 self.assertFloatsAlmostEqual(
140 math.fabs(self.record.get(self.xxErrKey)),
141 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6)
142 self.assertFloatsAlmostEqual(
143 math.fabs(self.record.get(self.yyErrKey)),
144 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6)
145 self.assertFloatsAlmostEqual(
146 math.fabs(self.record.get(self.xyErrKey)),
147 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6)
149 def testPersisted(self):
150 self.table.definePsfFlux("a")
151 self.table.defineCentroid("b")
152 self.table.defineShape("c")
153 with lsst.utils.tests.getTempFilePath(".fits") as filename:
154 self.catalog.writeFits(filename)
155 catalog = lsst.afw.table.SourceCatalog.readFits(filename)
156 table = catalog.getTable()
157 record = catalog[0]
158 # I'm using the keys from the non-persisted table. They should work at least in the
159 # current implementation
160 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux())
161 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag())
162 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b")
163 centroid = self.centroidKey.get(self.record)
164 self.assertEqual(centroid, record.getCentroid())
165 self.assertFloatsAlmostEqual(
166 math.fabs(self.record.get(self.xErrKey)),
167 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6)
168 self.assertFloatsAlmostEqual(
169 math.fabs(self.record.get(self.yErrKey)),
170 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6)
171 shape = self.shapeKey.get(self.record)
172 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c")
173 self.assertEqual(shape, record.getShape())
174 self.assertFloatsAlmostEqual(
175 math.fabs(self.record.get(self.xxErrKey)),
176 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6)
177 self.assertFloatsAlmostEqual(
178 math.fabs(self.record.get(self.yyErrKey)),
179 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6)
180 self.assertFloatsAlmostEqual(
181 math.fabs(self.record.get(self.xyErrKey)),
182 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6)
184 def testCanonical2(self):
185 self.table.definePsfFlux("a")
186 self.table.defineCentroid("b")
187 self.table.defineShape("c")
188 self.checkCanonical()
190 def testPickle(self):
191 p = pickle.dumps(self.catalog)
192 new = pickle.loads(p)
194 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames())
195 self.assertEqual(len(self.catalog), len(new))
196 for r1, r2 in zip(self.catalog, new):
197 # Columns that are easy to test
198 for field in ("a_instFlux", "a_instFluxErr", "id"):
199 k1 = self.catalog.schema.find(field).getKey()
200 k2 = new.schema.find(field).getKey()
201 self.assertEqual(r1[k1], r2[k2])
203 def testCoordUpdate(self):
204 self.table.defineCentroid("b")
205 wcs = makeWcs()
206 self.record.updateCoord(wcs)
207 coord1 = self.record.getCoord()
208 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey))
209 self.assertEqual(coord1, coord2)
211 def testSorting(self):
212 self.assertFalse(self.catalog.isSorted())
213 self.catalog.sort()
214 self.assertTrue(self.catalog.isSorted())
215 r = self.catalog.find(2)
216 self.assertEqual(r["id"], 2)
217 r = self.catalog.find(500)
218 self.assertIsNone(r)
220 def testConversion(self):
221 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog)
222 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog)
223 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True)
224 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True)
225 self.assertEqual(self.catalog.table, catalog1.table)
226 self.assertEqual(self.catalog.table, catalog2.table)
227 self.assertNotEqual(self.catalog.table, catalog3.table)
228 self.assertNotEqual(self.catalog.table, catalog3.table)
229 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4):
230 self.assertEqual(r, r1)
231 self.assertEqual(r, r2)
232 self.assertNotEqual(r, r3)
233 self.assertNotEqual(r, r4)
234 self.assertEqual(r.getId(), r3.getId())
235 self.assertEqual(r.getId(), r4.getId())
237 def testColumnView(self):
238 cols1 = self.catalog.getColumnView()
239 cols2 = self.catalog.columns
240 self.assertIs(cols1, cols2)
241 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView)
242 self.table.definePsfFlux("a")
243 self.table.defineCentroid("b")
244 self.table.defineShape("c")
245 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux())
246 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr())
247 self.assertFloatsEqual(cols2["b_x"], cols2.getX())
248 self.assertFloatsEqual(cols2["b_y"], cols2.getY())
249 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx())
250 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy())
251 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy())
253 # Trying to access slots which have been removed should raise.
254 self.catalog.table.schema.getAliasMap().erase("slot_Centroid")
255 self.catalog.table.schema.getAliasMap().erase("slot_Shape")
256 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy"]:
257 with self.assertRaises(lsst.pex.exceptions.LogicError):
258 getattr(self.catalog, f"get{quantity}")()
260 def testForwarding(self):
261 """Verify that Catalog forwards unknown methods to its table and/or columns."""
262 self.table.definePsfFlux("a")
263 self.table.defineCentroid("b")
264 self.table.defineShape("c")
265 self.assertFloatsEqual(self.catalog.columns["a_instFlux"],
266 self.catalog["a_instFlux"])
267 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey],
268 self.catalog.get(self.instFluxKey))
269 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey),
270 self.catalog.getPsfInstFlux())
271 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey())
272 with self.assertRaises(AttributeError):
273 self.catalog.foo()
275 def testBitsColumn(self):
277 allBits = self.catalog.getBits()
278 someBits = self.catalog.getBits(["a_flag", "c_flag"])
279 self.assertEqual(allBits.getMask("a_flag"), 0x1)
280 self.assertEqual(allBits.getMask("b_flag"), 0x2)
281 self.assertEqual(allBits.getMask("c_flag"), 0x4)
282 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1)
283 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2)
284 np.testing.assert_array_equal(
285 (allBits.array & 0x1 != 0), self.catalog.columns["a_flag"])
286 np.testing.assert_array_equal(
287 (allBits.array & 0x2 != 0), self.catalog.columns["b_flag"])
288 np.testing.assert_array_equal(
289 (allBits.array & 0x4 != 0), self.catalog.columns["c_flag"])
290 np.testing.assert_array_equal(
291 (someBits.array & 0x1 != 0), self.catalog.columns["a_flag"])
292 np.testing.assert_array_equal(
293 (someBits.array & 0x2 != 0), self.catalog.columns["c_flag"])
295 def testCast(self):
296 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog)
297 baseCat.cast(lsst.afw.table.SourceCatalog)
299 def testFootprints(self):
300 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS
301 '''
302 src1 = self.catalog.addNew()
303 src2 = self.catalog.addNew()
304 src3 = self.catalog.addNew()
305 self.fillRecord(src1)
306 self.fillRecord(src2)
307 self.fillRecord(src3)
308 src2.setParent(src1.getId())
310 W, H = 100, 100
311 mim = lsst.afw.image.MaskedImageF(W, H)
312 im = mim.getImage()
313 msk = mim.getMask()
314 var = mim.getVariance()
315 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int))
316 im.array[:] = y*1E6 + x*1E3
317 msk.array[:] = (y << 8) | x
318 var.array[:] = y*1E2 + x
319 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50)
320 circ = lsst.afw.detection.Footprint(spanSet)
321 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim)
322 src2.setFootprint(heavy)
324 for i, src in enumerate(self.catalog):
325 if src != src2:
326 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50)
327 src.setFootprint(lsst.afw.detection.Footprint(spanSet))
329 # insert this HeavyFootprint into an otherwise blank image (for comparing the results)
330 mim2 = lsst.afw.image.MaskedImageF(W, H)
331 heavy.insert(mim2)
333 with lsst.utils.tests.getTempFilePath(".fits") as fn:
334 self.catalog.writeFits(fn)
336 cat2 = lsst.afw.table.SourceCatalog.readFits(fn)
337 r2 = cat2[-2]
338 h2 = r2.getFootprint()
339 self.assertTrue(h2.isHeavy())
340 mim3 = lsst.afw.image.MaskedImageF(W, H)
341 h2.insert(mim3)
343 self.assertFalse(cat2[-1].getFootprint().isHeavy())
344 self.assertFalse(cat2[-3].getFootprint().isHeavy())
345 self.assertFalse(cat2[0].getFootprint().isHeavy())
346 self.assertFalse(cat2[1].getFootprint().isHeavy())
347 self.assertFalse(cat2[2].getFootprint().isHeavy())
349 if False:
350 # Write out before-n-after FITS images
351 for MI in [mim, mim2, mim3]:
352 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits')
353 os.close(f)
354 MI.writeFits(fn2)
355 print('wrote', fn2)
357 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray())
358 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray())
359 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray())
361 im3 = mim3.getImage()
362 ma3 = mim3.getMask()
363 va3 = mim3.getVariance()
364 for y in range(H):
365 for x in range(W):
366 if circ.contains(lsst.geom.Point2I(x, y)):
367 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT])
368 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT])
369 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT])
370 else:
371 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.)
372 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.)
373 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.)
375 cat3 = lsst.afw.table.SourceCatalog.readFits(
376 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
377 for src in cat3:
378 self.assertFalse(src.getFootprint().isHeavy())
379 cat4 = lsst.afw.table.SourceCatalog.readFits(
380 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
381 for src in cat4:
382 self.assertIsNone(src.getFootprint())
384 self.catalog.writeFits(
385 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
386 cat5 = lsst.afw.table.SourceCatalog.readFits(fn)
387 for src in cat5:
388 self.assertFalse(src.getFootprint().isHeavy())
390 self.catalog.writeFits(
391 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
392 cat6 = lsst.afw.table.SourceCatalog.readFits(fn)
393 for src in cat6:
394 self.assertIsNone(src.getFootprint())
396 def testIdFactory(self):
397 expId = int(1257198)
398 reserved = 32
399 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved)
400 id1 = factory()
401 id2 = factory()
402 self.assertEqual(id2 - id1, 1)
403 factory.notify(0xFFFFFFFF)
404 with self.assertRaises(lsst.pex.exceptions.LengthError):
405 factory()
406 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
407 factory.notify(0x1FFFFFFFF)
408 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
409 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved)
411 def testFamilies(self):
412 self.catalog.sort()
413 parents = self.catalog.getChildren(0)
414 self.assertEqual(list(parents), list(self.catalog))
415 parentKey = lsst.afw.table.SourceTable.getParentKey()
416 for parent in parents:
417 self.assertEqual(parent.get(parentKey), 0)
418 for i in range(10):
419 child = self.catalog.addNew()
420 self.fillRecord(child)
421 child.set(parentKey, parent.getId())
422 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents],
423 [record.getId() for record in self.catalog])
424 for parent, (children, ids) in zip(parents, childrenIter):
425 self.assertEqual(len(children), 10)
426 self.assertEqual(len(children), len(ids))
427 for child, id in zip(children, ids):
428 self.assertEqual(child.getParent(), parent.getId())
429 self.assertEqual(child.getId(), id)
431 # Check detection of unsorted catalog
432 self.catalog.sort(self.instFluxKey)
433 with self.assertRaises(AssertionError):
434 self.catalog.getChildren(0)
435 self.catalog.sort(parentKey)
436 self.catalog.getChildren(0) # Just care this succeeds
438 def testFitsReadVersion0Compatibility(self):
439 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits"))
440 self.assertTrue(cat.getPsfFluxSlot().isValid())
441 self.assertTrue(cat.getApFluxSlot().isValid())
442 self.assertTrue(cat.getGaussianFluxSlot().isValid())
443 self.assertTrue(cat.getModelFluxSlot().isValid())
444 self.assertTrue(cat.getCentroidSlot().isValid())
445 self.assertTrue(cat.getShapeSlot().isValid())
446 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(),
447 cat.schema.find("flux_psf").key)
448 self.assertEqual(cat.getApFluxSlot().getMeasKey(),
449 cat.schema.find("flux_sinc").key)
450 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(),
451 cat.schema.find("flux_naive").key)
452 self.assertEqual(cat.getModelFluxSlot().getMeasKey(),
453 cat.schema.find("cmodel_flux").key)
454 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(),
455 cat.schema.find("centroid_sdss_x").key)
456 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(),
457 cat.schema.find("centroid_sdss_y").key)
458 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(),
459 cat.schema.find("shape_hsm_moments_xx").key)
460 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(),
461 cat.schema.find("shape_hsm_moments_yy").key)
462 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(),
463 cat.schema.find("shape_hsm_moments_xy").key)
464 self.assertEqual(cat.getPsfFluxSlot().getErrKey(),
465 cat.schema.find("flux_psf_err").key)
466 self.assertEqual(cat.getApFluxSlot().getErrKey(),
467 cat.schema.find("flux_sinc_err").key)
468 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(),
469 cat.schema.find("flux_naive_err").key)
470 self.assertEqual(cat.getModelFluxSlot().getErrKey(),
471 cat.schema.find("cmodel_flux_err").key)
472 self.assertEqual(
473 cat.getCentroidSlot().getErrKey(),
474 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"]))
475 self.assertEqual(
476 cat.getShapeSlot().getErrKey(),
477 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"]))
478 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(),
479 cat.schema.find("flux_psf_flags").key)
480 self.assertEqual(cat.getApFluxSlot().getFlagKey(),
481 cat.schema.find("flux_sinc_flags").key)
482 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(),
483 cat.schema.find("flux_naive_flags").key)
484 self.assertEqual(cat.getModelFluxSlot().getFlagKey(),
485 cat.schema.find("cmodel_flux_flags").key)
486 self.assertEqual(cat.getCentroidSlot().getFlagKey(),
487 cat.schema.find("centroid_sdss_flags").key)
488 self.assertEqual(cat.getShapeSlot().getFlagKey(),
489 cat.schema.find("shape_hsm_moments_flags").key)
491 def testFitsReadVersion1Compatibility(self):
492 """Test reading of catalogs with version 1 schema
494 Version 1 catalogs need to have added aliases from Sigma->Err and
495 from `_flux`->`_instFlux`.
496 """
497 cat = lsst.afw.table.SourceCatalog.readFits(
498 os.path.join(testPath, "data", "sourceTable-v1.fits"))
499 self.assertEqual(
500 cat.getCentroidSlot().getErrKey(),
501 lsst.afw.table.CovarianceMatrix2fKey(
502 cat.schema["slot_Centroid"],
503 ["x", "y"]))
504 self.assertEqual(
505 cat.getShapeSlot().getErrKey(),
506 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"]))
507 # check the flux->instFlux conversion
508 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey())
509 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey())
511 def testFitsReadVersion2CompatibilityRealSourceCatalog(self):
512 """DM-15891: some fields were getting aliases they shouldn't have."""
513 cat = lsst.afw.table.SourceCatalog.readFits(
514 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits"))
515 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema)
516 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema)
517 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema)
519 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self):
520 """DM-16068: some fields were not getting aliases they should have
522 In particular, the alias setting relies on flux fields having their
523 units set properly. Prior to a resolution of DM-16068, the units
524 were not getting set for several CModel flux fields and one deblender
525 field (deblend_psfFlux), and thus were not getting the
526 `_flux`->`_instFlux` aliases set.
528 NOTE: this test will (is meant to) fail on the read until DM-16068 is
529 resolved. The error is:
531 lsst::pex::exceptions::NotFoundError: 'Field or subfield with
532 name 'modelfit_CModel_instFlux' not found with type 'D'.'
533 """
534 cat = lsst.afw.table.SourceCatalog.readFits(
535 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits"))
536 self.assertIn('modelfit_CModel_instFlux', cat.schema)
537 self.assertIn('modelfit_CModel_instFluxErr', cat.schema)
538 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema)
539 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema)
540 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema)
541 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema)
543 def testFitsVersion2Compatibility(self):
544 """Test reading of catalogs with version 2 schema
546 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`.
547 """
548 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits"))
549 # check the flux->instFlux conversion
550 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey())
551 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey())
553 def testDM1083(self):
554 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
555 st = lsst.afw.table.SourceTable.make(schema)
556 cat = lsst.afw.table.SourceCatalog(st)
557 tmp = lsst.afw.table.SourceCatalog(cat.getTable())
558 record = tmp.addNew()
559 cat.extend(tmp)
560 self.assertEqual(cat[0].getId(), record.getId())
561 # check that the same record is in both catalogs (not a copy)
562 record.setId(15)
563 self.assertEqual(cat[0].getId(), record.getId())
565 def testSlotUndefine(self):
566 """Test that we can correctly define and undefine a slot after a SourceTable has been created"""
567 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
568 key = schema.addField("a_instFlux", type=np.float64, doc="flux field")
569 table = lsst.afw.table.SourceTable.make(schema)
570 table.definePsfFlux("a")
571 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key)
572 table.schema.getAliasMap().erase("slot_PsfFlux")
573 self.assertFalse(table.getPsfFluxSlot().isValid())
575 def testOldFootprintPersistence(self):
576 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older
577 version of the pipeline with a different format.
578 """
579 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits")
580 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename)
581 self.assertEqual(len(catalog1), 2)
582 with self.assertRaises(LookupError):
583 catalog1.schema.find("footprint")
584 fp1 = catalog1[0].getFootprint()
585 fp2 = catalog1[1].getFootprint()
586 self.assertEqual(fp1.getArea(), 495)
587 self.assertEqual(fp2.getArea(), 767)
588 self.assertFalse(fp1.isHeavy())
589 self.assertTrue(fp2.isHeavy())
590 self.assertEqual(len(fp1.getSpans()), 29)
591 self.assertEqual(len(fp2.getSpans()), 44)
592 self.assertEqual(len(fp1.getPeaks()), 1)
593 self.assertEqual(len(fp2.getPeaks()), 1)
594 self.assertEqual(fp1.getBBox(),
595 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29)))
596 self.assertEqual(fp2.getBBox(),
597 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38)))
598 hfp = lsst.afw.detection.HeavyFootprintF(fp2)
599 self.assertEqual(len(hfp.getImageArray()), fp2.getArea())
600 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea())
601 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea())
602 catalog2 = lsst.afw.table.SourceCatalog.readFits(
603 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
604 self.assertEqual(list(fp1.getSpans()),
605 list(catalog2[0].getFootprint().getSpans()))
606 self.assertEqual(list(fp2.getSpans()),
607 list(catalog2[1].getFootprint().getSpans()))
608 self.assertFalse(catalog2[1].getFootprint().isHeavy())
609 catalog3 = lsst.afw.table.SourceCatalog.readFits(
610 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
611 self.assertEqual(catalog3[0].getFootprint(), None)
612 self.assertEqual(catalog3[1].getFootprint(), None)
614 def _testFluxSlot(self, slotName):
615 """Demonstrate that we can create & use the named Flux slot."""
616 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
617 baseName = "afw_Test"
618 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux")
619 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty")
620 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag")
621 catalog = lsst.afw.table.SourceCatalog(schema)
622 table = catalog.table
624 # Initially, the slot is undefined.
625 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid())
627 # After definition, it maps to the keys defined above.
628 getattr(table, f"define{slotName}")(baseName)
629 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid())
630 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey)
631 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey)
632 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey)
634 # We should be able to retrieve arbitrary values set in records.
635 record = catalog.addNew()
636 instFlux, err, flag = 10.0, 1.0, False
637 record.set(instFluxKey, instFlux)
638 record.set(errKey, err)
639 record.set(flagKey, flag)
640 instFluxName = slotName.replace("Flux", "InstFlux")
641 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux)
642 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err)
643 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag)
645 # Should also be able to retrieve them as columns from the catalog
646 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux)
647 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err)
649 # And we should be able to delete the slot, breaking the mapping.
650 table.schema.getAliasMap().erase(f"slot_{slotName}")
651 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid())
652 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey)
653 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey)
654 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey)
656 # When the slot has been deleted, attempting to access it should
657 # throw a LogicError.
658 with self.assertRaises(lsst.pex.exceptions.LogicError):
659 getattr(catalog, f"get{instFluxName}")()
660 with self.assertRaises(lsst.pex.exceptions.LogicError):
661 getattr(catalog, f"get{instFluxName}Err")()
663 def testFluxSlots(self):
664 """Check that all the expected flux slots are present & correct."""
665 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux",
666 "PsfFlux"]:
667 self._testFluxSlot(slotName)
669 # But, of course, we should not accept a slot which hasn't be defined.
670 with self.assertRaises(AttributeError):
671 self._testFluxSlot("NotExtantFlux")
673 def testStr(self):
674 """Check that the str() produced on a catalog contains expected things."""
675 string = str(self.catalog)
676 for field in ('id', 'coord_ra', 'coord_dec'):
677 self.assertIn(field, string)
679 def testRepr(self):
680 """Check that the repr() produced on a catalog contains expected things."""
681 string = repr(self.catalog)
682 self.assertIn(str(type(self.catalog)), string)
683 for field in ('id', 'coord_ra', 'coord_dec'):
684 self.assertIn(field, string)
686 def testStrNonContiguous(self):
687 """Check that str() doesn't fail on non-contiguous tables."""
688 del self.catalog[1]
689 string = str(self.catalog)
690 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string)
691 for field in ('id', 'coord_ra', 'coord_dec'):
692 self.assertIn(field, string)
694 def testRecordStr(self):
695 """Test that str(record) contains expected things."""
696 string = str(self.catalog[0])
697 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'):
698 self.assertIn(field, string)
700 def testRecordRepr(self):
701 """Test that repr(record) contains expected things."""
702 string = repr(self.catalog[0])
703 self.assertIn(str(type(self.catalog[0])), string)
704 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'):
705 self.assertIn(field, string)
707 def testGetNonContiguous(self):
708 """Check that we can index on non-contiguous tables"""
709 # Make a non-contiguous catalog
710 nonContiguous = type(self.catalog)(self.catalog.table)
711 for rr in reversed(self.catalog):
712 nonContiguous.append(rr)
713 num = len(self.catalog)
714 # Check assumptions
715 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog
716 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique
717 # Indexing with boolean array
718 select = np.zeros(num, dtype=bool)
719 select[1] = True
720 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"])
721 # Extracting a number column
722 column = "a_instFlux"
723 array = nonContiguous[column]
724 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column])
725 with self.assertRaises(ValueError):
726 array[1] = 1.2345 # Should be immutable
727 # Extracting a flag column
728 column = "a_flag"
729 array = nonContiguous[column]
730 np.testing.assert_equal(np.flip(array, 0), self.catalog[column])
731 with self.assertRaises(ValueError):
732 array[1] = True # Should be immutable
735class MemoryTester(lsst.utils.tests.MemoryTestCase):
736 pass
739def setup_module(module):
740 lsst.utils.tests.init()
743if __name__ == "__main__": 743 ↛ 744line 743 didn't jump to line 744, because the condition on line 743 was never true
744 lsst.utils.tests.init()
745 unittest.main()