Coverage for tests/test_sourceTable.py: 10%
509 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-08 03:13 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-08 03:13 -0700
1# This file is part of afw.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Tests for table.SourceTable
25Run with:
26 python test_sourceTable.py
27or
28 pytest test_sourceTable.py
29"""
30import os
31import unittest
32import tempfile
33import pickle
34import math
36import numpy as np
38import lsst.utils.tests
39import lsst.pex.exceptions
40import lsst.geom
41import lsst.afw.table
42import lsst.afw.geom
43import lsst.afw.image
44import lsst.afw.detection
46testPath = os.path.abspath(os.path.dirname(__file__))
49def makeArray(size, dtype):
50 return np.array(np.random.randn(*size), dtype=dtype)
53def makeCov(size, dtype):
54 m = np.array(np.random.randn(size, size), dtype=dtype)
55 return np.dot(m, m.transpose())
58def makeWcs():
59 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees,
60 5.090329*lsst.geom.degrees)
61 crpix = lsst.geom.Point2D(2036.0, 2000.0)
62 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5])
63 cdMatrix.shape = (2, 2)
64 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix)
67class SourceTableTestCase(lsst.utils.tests.TestCase):
69 def fillRecord(self, record):
70 record.set(self.instFluxKey, np.random.randn())
71 record.set(self.instFluxErrKey, np.random.randn())
72 record.set(self.centroidKey.getX(), np.random.randn())
73 record.set(self.centroidKey.getY(), np.random.randn())
74 record.set(self.xErrKey, np.random.randn())
75 record.set(self.yErrKey, np.random.randn())
76 record.set(self.shapeKey.getIxx(), np.random.randn())
77 record.set(self.shapeKey.getIyy(), np.random.randn())
78 record.set(self.shapeKey.getIxy(), np.random.randn())
79 record.set(self.xxErrKey, np.random.randn())
80 record.set(self.yyErrKey, np.random.randn())
81 record.set(self.xyErrKey, np.random.randn())
82 record.set(self.psfShapeKey.getIxx(), np.random.randn())
83 record.set(self.psfShapeKey.getIyy(), np.random.randn())
84 record.set(self.psfShapeKey.getIxy(), np.random.randn())
85 record.set(self.fluxFlagKey, np.random.randn() > 0)
86 record.set(self.centroidFlagKey, np.random.randn() > 0)
87 record.set(self.shapeFlagKey, np.random.randn() > 0)
89 def setUp(self):
90 np.random.seed(1)
91 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema()
92 self.instFluxKey = self.schema.addField("a_instFlux", type="D")
93 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D")
94 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag")
96 # the meas field is added using a functor key, but the error is added
97 # as scalars, as we lack a ResultKey functor as exists in meas_base
98 self.centroidKey = lsst.afw.table.Point2DKey.addFields(
99 self.schema, "b", "", "pixel")
100 self.xErrKey = self.schema.addField("b_xErr", type="F")
101 self.yErrKey = self.schema.addField("b_yErr", type="F")
102 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag")
104 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields(
105 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL)
106 self.xxErrKey = self.schema.addField("c_xxErr", type="F")
107 self.xyErrKey = self.schema.addField("c_xyErr", type="F")
108 self.yyErrKey = self.schema.addField("c_yyErr", type="F")
109 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag")
111 self.psfShapeKey = lsst.afw.table.QuadrupoleKey.addFields(
112 self.schema, "d", "", lsst.afw.table.CoordinateType.PIXEL)
113 self.psfShapeFlagKey = self.schema.addField("d_flag", type="Flag")
115 self.table = lsst.afw.table.SourceTable.make(self.schema)
116 self.catalog = lsst.afw.table.SourceCatalog(self.table)
117 self.record = self.catalog.addNew()
118 self.fillRecord(self.record)
119 self.record.setId(50)
120 self.fillRecord(self.catalog.addNew())
121 self.fillRecord(self.catalog.addNew())
123 def tearDown(self):
124 del self.schema
125 del self.record
126 del self.table
127 del self.catalog
129 def checkCanonical(self):
130 self.assertEqual(self.record.get(self.instFluxKey),
131 self.record.getPsfInstFlux())
132 self.assertEqual(self.record.get(self.fluxFlagKey),
133 self.record.getPsfFluxFlag())
134 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b")
135 self.assertEqual(self.centroidKey.get(self.record),
136 self.record.getCentroid())
137 self.assertFloatsAlmostEqual(
138 math.fabs(self.record.get(self.xErrKey)),
139 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6)
140 self.assertFloatsAlmostEqual(
141 math.fabs(self.record.get(self.yErrKey)),
142 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6)
143 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c")
144 self.assertEqual(self.shapeKey.get(self.record),
145 self.record.getShape())
146 self.assertFloatsAlmostEqual(
147 math.fabs(self.record.get(self.xxErrKey)),
148 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6)
149 self.assertFloatsAlmostEqual(
150 math.fabs(self.record.get(self.yyErrKey)),
151 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6)
152 self.assertFloatsAlmostEqual(
153 math.fabs(self.record.get(self.xyErrKey)),
154 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6)
155 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_PsfShape"), "d")
156 self.assertEqual(self.psfShapeKey.get(self.record),
157 self.record.getPsfShape())
159 def testPersisted(self):
160 self.table.definePsfFlux("a")
161 self.table.defineCentroid("b")
162 self.table.defineShape("c")
163 self.table.definePsfShape("d")
164 with lsst.utils.tests.getTempFilePath(".fits") as filename:
165 self.catalog.writeFits(filename)
166 catalog = lsst.afw.table.SourceCatalog.readFits(filename)
167 table = catalog.getTable()
168 record = catalog[0]
169 # I'm using the keys from the non-persisted table. They should work at least in the
170 # current implementation
171 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux())
172 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag())
173 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b")
174 centroid = self.centroidKey.get(self.record)
175 self.assertEqual(centroid, record.getCentroid())
176 self.assertFloatsAlmostEqual(
177 math.fabs(self.record.get(self.xErrKey)),
178 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6)
179 self.assertFloatsAlmostEqual(
180 math.fabs(self.record.get(self.yErrKey)),
181 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6)
182 shape = self.shapeKey.get(self.record)
183 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c")
184 self.assertEqual(shape, record.getShape())
185 self.assertFloatsAlmostEqual(
186 math.fabs(self.record.get(self.xxErrKey)),
187 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6)
188 self.assertFloatsAlmostEqual(
189 math.fabs(self.record.get(self.yyErrKey)),
190 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6)
191 self.assertFloatsAlmostEqual(
192 math.fabs(self.record.get(self.xyErrKey)),
193 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6)
194 psfShape = self.psfShapeKey.get(self.record)
195 self.assertEqual(table.getSchema().getAliasMap().get("slot_PsfShape"), "d")
196 self.assertEqual(psfShape, record.getPsfShape())
198 def testCanonical2(self):
199 self.table.definePsfFlux("a")
200 self.table.defineCentroid("b")
201 self.table.defineShape("c")
202 self.table.definePsfShape("d")
203 self.checkCanonical()
205 def testPickle(self):
206 p = pickle.dumps(self.catalog)
207 new = pickle.loads(p)
209 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames())
210 self.assertEqual(len(self.catalog), len(new))
211 for r1, r2 in zip(self.catalog, new):
212 # Columns that are easy to test
213 for field in ("a_instFlux", "a_instFluxErr", "id"):
214 k1 = self.catalog.schema.find(field).getKey()
215 k2 = new.schema.find(field).getKey()
216 self.assertEqual(r1[k1], r2[k2])
218 def testCoordUpdate(self):
219 self.table.defineCentroid("b")
220 wcs = makeWcs()
221 self.record.updateCoord(wcs)
222 coord1 = self.record.getCoord()
223 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey))
224 self.assertEqual(coord1, coord2)
226 def testSorting(self):
227 self.assertFalse(self.catalog.isSorted())
228 self.catalog.sort()
229 self.assertTrue(self.catalog.isSorted())
230 r = self.catalog.find(2)
231 self.assertEqual(r["id"], 2)
232 r = self.catalog.find(500)
233 self.assertIsNone(r)
235 def testConversion(self):
236 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog)
237 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog)
238 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True)
239 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True)
240 self.assertEqual(self.catalog.table, catalog1.table)
241 self.assertEqual(self.catalog.table, catalog2.table)
242 self.assertNotEqual(self.catalog.table, catalog3.table)
243 self.assertNotEqual(self.catalog.table, catalog3.table)
244 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4):
245 self.assertEqual(r, r1)
246 self.assertEqual(r, r2)
247 self.assertNotEqual(r, r3)
248 self.assertNotEqual(r, r4)
249 self.assertEqual(r.getId(), r3.getId())
250 self.assertEqual(r.getId(), r4.getId())
252 def testColumnView(self):
253 cols1 = self.catalog.getColumnView()
254 cols2 = self.catalog.columns
255 self.assertIs(cols1, cols2)
256 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView)
257 self.table.definePsfFlux("a")
258 self.table.defineCentroid("b")
259 self.table.defineShape("c")
260 self.table.definePsfShape("d")
261 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux())
262 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr())
263 self.assertFloatsEqual(cols2["b_x"], cols2.getX())
264 self.assertFloatsEqual(cols2["b_y"], cols2.getY())
265 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx())
266 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy())
267 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy())
268 self.assertFloatsEqual(cols2["d_xx"], cols2.getPsfIxx())
269 self.assertFloatsEqual(cols2["d_yy"], cols2.getPsfIyy())
270 self.assertFloatsEqual(cols2["d_xy"], cols2.getPsfIxy())
272 # Trying to access slots which have been removed should raise.
273 self.catalog.table.schema.getAliasMap().erase("slot_Centroid")
274 self.catalog.table.schema.getAliasMap().erase("slot_Shape")
275 self.catalog.table.schema.getAliasMap().erase("slot_PsfShape")
276 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy", "PsfIxx", "PsfIyy", "PsfIxy"]:
277 with self.assertRaises(lsst.pex.exceptions.LogicError):
278 getattr(self.catalog, f"get{quantity}")()
280 def testForwarding(self):
281 """Verify that Catalog forwards unknown methods to its table and/or columns."""
282 self.table.definePsfFlux("a")
283 self.table.defineCentroid("b")
284 self.table.defineShape("c")
285 self.assertFloatsEqual(self.catalog.columns["a_instFlux"],
286 self.catalog["a_instFlux"])
287 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey],
288 self.catalog.get(self.instFluxKey))
289 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey),
290 self.catalog.getPsfInstFlux())
291 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey())
292 with self.assertRaises(AttributeError):
293 self.catalog.foo()
295 def testBitsColumn(self):
297 allBits = self.catalog.getBits()
298 someBits = self.catalog.getBits(["a_flag", "c_flag"])
299 self.assertEqual(allBits.getMask("a_flag"), 0x1)
300 self.assertEqual(allBits.getMask("b_flag"), 0x2)
301 self.assertEqual(allBits.getMask("c_flag"), 0x4)
302 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1)
303 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2)
304 np.testing.assert_array_equal((allBits.array & 0x1 != 0), self.catalog["a_flag"])
305 np.testing.assert_array_equal((allBits.array & 0x2 != 0), self.catalog["b_flag"])
306 np.testing.assert_array_equal((allBits.array & 0x4 != 0), self.catalog["c_flag"])
307 np.testing.assert_array_equal((someBits.array & 0x1 != 0), self.catalog["a_flag"])
308 np.testing.assert_array_equal((someBits.array & 0x2 != 0), self.catalog["c_flag"])
310 def testCast(self):
311 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog)
312 baseCat.cast(lsst.afw.table.SourceCatalog)
314 def testFootprints(self):
315 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS
316 '''
317 src1 = self.catalog.addNew()
318 src2 = self.catalog.addNew()
319 src3 = self.catalog.addNew()
320 self.fillRecord(src1)
321 self.fillRecord(src2)
322 self.fillRecord(src3)
323 src2.setParent(src1.getId())
325 W, H = 100, 100
326 mim = lsst.afw.image.MaskedImageF(W, H)
327 im = mim.getImage()
328 msk = mim.getMask()
329 var = mim.getVariance()
330 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int))
331 im.array[:] = y*1E6 + x*1E3
332 msk.array[:] = (y << 8) | x
333 var.array[:] = y*1E2 + x
334 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50)
335 circ = lsst.afw.detection.Footprint(spanSet)
336 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim)
337 src2.setFootprint(heavy)
339 for i, src in enumerate(self.catalog):
340 if src != src2:
341 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50)
342 src.setFootprint(lsst.afw.detection.Footprint(spanSet))
344 # insert this HeavyFootprint into an otherwise blank image (for comparing the results)
345 mim2 = lsst.afw.image.MaskedImageF(W, H)
346 heavy.insert(mim2)
348 with lsst.utils.tests.getTempFilePath(".fits") as fn:
349 self.catalog.writeFits(fn)
351 cat2 = lsst.afw.table.SourceCatalog.readFits(fn)
352 r2 = cat2[-2]
353 h2 = r2.getFootprint()
354 self.assertTrue(h2.isHeavy())
355 mim3 = lsst.afw.image.MaskedImageF(W, H)
356 h2.insert(mim3)
358 self.assertFalse(cat2[-1].getFootprint().isHeavy())
359 self.assertFalse(cat2[-3].getFootprint().isHeavy())
360 self.assertFalse(cat2[0].getFootprint().isHeavy())
361 self.assertFalse(cat2[1].getFootprint().isHeavy())
362 self.assertFalse(cat2[2].getFootprint().isHeavy())
364 if False:
365 # Write out before-n-after FITS images
366 for MI in [mim, mim2, mim3]:
367 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits')
368 os.close(f)
369 MI.writeFits(fn2)
370 print('wrote', fn2)
372 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray())
373 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray())
374 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray())
376 im3 = mim3.getImage()
377 ma3 = mim3.getMask()
378 va3 = mim3.getVariance()
379 for y in range(H):
380 for x in range(W):
381 if circ.contains(lsst.geom.Point2I(x, y)):
382 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT])
383 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT])
384 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT])
385 else:
386 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.)
387 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.)
388 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.)
390 cat3 = lsst.afw.table.SourceCatalog.readFits(
391 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
392 for src in cat3:
393 self.assertFalse(src.getFootprint().isHeavy())
394 cat4 = lsst.afw.table.SourceCatalog.readFits(
395 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
396 for src in cat4:
397 self.assertIsNone(src.getFootprint())
399 self.catalog.writeFits(
400 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
401 cat5 = lsst.afw.table.SourceCatalog.readFits(fn)
402 for src in cat5:
403 self.assertFalse(src.getFootprint().isHeavy())
405 self.catalog.writeFits(
406 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
407 cat6 = lsst.afw.table.SourceCatalog.readFits(fn)
408 for src in cat6:
409 self.assertIsNone(src.getFootprint())
411 def testIdFactory(self):
412 expId = int(1257198)
413 reserved = 32
414 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved)
415 id1 = factory()
416 id2 = factory()
417 self.assertEqual(id2 - id1, 1)
418 factory.notify(0xFFFFFFFF)
419 with self.assertRaises(lsst.pex.exceptions.LengthError):
420 factory()
421 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
422 factory.notify(0x1FFFFFFFF)
423 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError):
424 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved)
426 def testFamilies(self):
427 self.catalog.sort()
428 parents = self.catalog.getChildren(0)
429 self.assertEqual(list(parents), list(self.catalog))
430 parentKey = lsst.afw.table.SourceTable.getParentKey()
431 for parent in parents:
432 self.assertEqual(parent.get(parentKey), 0)
433 for i in range(10):
434 child = self.catalog.addNew()
435 self.fillRecord(child)
436 child.set(parentKey, parent.getId())
437 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents],
438 [record.getId() for record in self.catalog])
439 for parent, (children, ids) in zip(parents, childrenIter):
440 self.assertEqual(len(children), 10)
441 self.assertEqual(len(children), len(ids))
442 for child, id in zip(children, ids):
443 self.assertEqual(child.getParent(), parent.getId())
444 self.assertEqual(child.getId(), id)
446 # Check detection of unsorted catalog
447 self.catalog.sort(self.instFluxKey)
448 with self.assertRaises(AssertionError):
449 self.catalog.getChildren(0)
450 self.catalog.sort(parentKey)
451 self.catalog.getChildren(0) # Just care this succeeds
453 def testFitsReadVersion0Compatibility(self):
454 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits"))
455 self.assertTrue(cat.getPsfFluxSlot().isValid())
456 self.assertTrue(cat.getApFluxSlot().isValid())
457 self.assertTrue(cat.getGaussianFluxSlot().isValid())
458 self.assertTrue(cat.getModelFluxSlot().isValid())
459 self.assertTrue(cat.getCentroidSlot().isValid())
460 self.assertTrue(cat.getShapeSlot().isValid())
461 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(),
462 cat.schema.find("flux_psf").key)
463 self.assertEqual(cat.getApFluxSlot().getMeasKey(),
464 cat.schema.find("flux_sinc").key)
465 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(),
466 cat.schema.find("flux_naive").key)
467 self.assertEqual(cat.getModelFluxSlot().getMeasKey(),
468 cat.schema.find("cmodel_flux").key)
469 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(),
470 cat.schema.find("centroid_sdss_x").key)
471 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(),
472 cat.schema.find("centroid_sdss_y").key)
473 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(),
474 cat.schema.find("shape_hsm_moments_xx").key)
475 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(),
476 cat.schema.find("shape_hsm_moments_yy").key)
477 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(),
478 cat.schema.find("shape_hsm_moments_xy").key)
479 self.assertEqual(cat.getPsfFluxSlot().getErrKey(),
480 cat.schema.find("flux_psf_err").key)
481 self.assertEqual(cat.getApFluxSlot().getErrKey(),
482 cat.schema.find("flux_sinc_err").key)
483 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(),
484 cat.schema.find("flux_naive_err").key)
485 self.assertEqual(cat.getModelFluxSlot().getErrKey(),
486 cat.schema.find("cmodel_flux_err").key)
487 self.assertEqual(
488 cat.getCentroidSlot().getErrKey(),
489 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"]))
490 self.assertEqual(
491 cat.getShapeSlot().getErrKey(),
492 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"]))
493 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(),
494 cat.schema.find("flux_psf_flags").key)
495 self.assertEqual(cat.getApFluxSlot().getFlagKey(),
496 cat.schema.find("flux_sinc_flags").key)
497 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(),
498 cat.schema.find("flux_naive_flags").key)
499 self.assertEqual(cat.getModelFluxSlot().getFlagKey(),
500 cat.schema.find("cmodel_flux_flags").key)
501 self.assertEqual(cat.getCentroidSlot().getFlagKey(),
502 cat.schema.find("centroid_sdss_flags").key)
503 self.assertEqual(cat.getShapeSlot().getFlagKey(),
504 cat.schema.find("shape_hsm_moments_flags").key)
506 def testFitsReadVersion1Compatibility(self):
507 """Test reading of catalogs with version 1 schema
509 Version 1 catalogs need to have added aliases from Sigma->Err and
510 from `_flux`->`_instFlux`.
511 """
512 cat = lsst.afw.table.SourceCatalog.readFits(
513 os.path.join(testPath, "data", "sourceTable-v1.fits"))
514 self.assertEqual(
515 cat.getCentroidSlot().getErrKey(),
516 lsst.afw.table.CovarianceMatrix2fKey(
517 cat.schema["slot_Centroid"],
518 ["x", "y"]))
519 self.assertEqual(
520 cat.getShapeSlot().getErrKey(),
521 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"]))
522 # check the flux->instFlux conversion
523 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey())
524 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey())
526 def testFitsReadVersion2CompatibilityRealSourceCatalog(self):
527 """DM-15891: some fields were getting aliases they shouldn't have."""
528 cat = lsst.afw.table.SourceCatalog.readFits(
529 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits"))
530 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema)
531 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema)
532 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema)
534 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self):
535 """DM-16068: some fields were not getting aliases they should have
537 In particular, the alias setting relies on flux fields having their
538 units set properly. Prior to a resolution of DM-16068, the units
539 were not getting set for several CModel flux fields and one deblender
540 field (deblend_psfFlux), and thus were not getting the
541 `_flux`->`_instFlux` aliases set.
543 NOTE: this test will (is meant to) fail on the read until DM-16068 is
544 resolved. The error is:
546 lsst::pex::exceptions::NotFoundError: 'Field or subfield with
547 name 'modelfit_CModel_instFlux' not found with type 'D'.'
548 """
549 cat = lsst.afw.table.SourceCatalog.readFits(
550 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits"))
551 self.assertIn('modelfit_CModel_instFlux', cat.schema)
552 self.assertIn('modelfit_CModel_instFluxErr', cat.schema)
553 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema)
554 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema)
555 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema)
556 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema)
558 def testFitsVersion2Compatibility(self):
559 """Test reading of catalogs with version 2 schema
561 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`.
562 """
563 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits"))
564 # check the flux->instFlux conversion
565 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey())
566 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey())
568 def testDM1083(self):
569 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
570 st = lsst.afw.table.SourceTable.make(schema)
571 cat = lsst.afw.table.SourceCatalog(st)
572 tmp = lsst.afw.table.SourceCatalog(cat.getTable())
573 record = tmp.addNew()
574 cat.extend(tmp)
575 self.assertEqual(cat[0].getId(), record.getId())
576 # check that the same record is in both catalogs (not a copy)
577 record.setId(15)
578 self.assertEqual(cat[0].getId(), record.getId())
580 def testSlotUndefine(self):
581 """Test that we can correctly define and undefine a slot after a SourceTable has been created"""
582 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
583 key = schema.addField("a_instFlux", type=np.float64, doc="flux field")
584 table = lsst.afw.table.SourceTable.make(schema)
585 table.definePsfFlux("a")
586 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key)
587 table.schema.getAliasMap().erase("slot_PsfFlux")
588 self.assertFalse(table.getPsfFluxSlot().isValid())
590 def testOldFootprintPersistence(self):
591 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older
592 version of the pipeline with a different format.
593 """
594 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits")
595 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename)
596 self.assertEqual(len(catalog1), 2)
597 with self.assertRaises(LookupError):
598 catalog1.schema.find("footprint")
599 fp1 = catalog1[0].getFootprint()
600 fp2 = catalog1[1].getFootprint()
601 self.assertEqual(fp1.getArea(), 495)
602 self.assertEqual(fp2.getArea(), 767)
603 self.assertFalse(fp1.isHeavy())
604 self.assertTrue(fp2.isHeavy())
605 self.assertEqual(len(fp1.getSpans()), 29)
606 self.assertEqual(len(fp2.getSpans()), 44)
607 self.assertEqual(len(fp1.getPeaks()), 1)
608 self.assertEqual(len(fp2.getPeaks()), 1)
609 self.assertEqual(fp1.getBBox(),
610 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29)))
611 self.assertEqual(fp2.getBBox(),
612 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38)))
613 hfp = lsst.afw.detection.HeavyFootprintF(fp2)
614 self.assertEqual(len(hfp.getImageArray()), fp2.getArea())
615 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea())
616 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea())
617 catalog2 = lsst.afw.table.SourceCatalog.readFits(
618 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS)
619 self.assertEqual(list(fp1.getSpans()),
620 list(catalog2[0].getFootprint().getSpans()))
621 self.assertEqual(list(fp2.getSpans()),
622 list(catalog2[1].getFootprint().getSpans()))
623 self.assertFalse(catalog2[1].getFootprint().isHeavy())
624 catalog3 = lsst.afw.table.SourceCatalog.readFits(
625 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS)
626 self.assertEqual(catalog3[0].getFootprint(), None)
627 self.assertEqual(catalog3[1].getFootprint(), None)
629 def _testFluxSlot(self, slotName):
630 """Demonstrate that we can create & use the named Flux slot."""
631 schema = lsst.afw.table.SourceTable.makeMinimalSchema()
632 baseName = "afw_Test"
633 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux")
634 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty")
635 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag")
636 catalog = lsst.afw.table.SourceCatalog(schema)
637 table = catalog.table
639 # Initially, the slot is undefined.
640 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid())
642 # After definition, it maps to the keys defined above.
643 getattr(table, f"define{slotName}")(baseName)
644 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid())
645 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey)
646 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey)
647 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey)
649 # We should be able to retrieve arbitrary values set in records.
650 record = catalog.addNew()
651 instFlux, err, flag = 10.0, 1.0, False
652 record.set(instFluxKey, instFlux)
653 record.set(errKey, err)
654 record.set(flagKey, flag)
655 instFluxName = slotName.replace("Flux", "InstFlux")
656 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux)
657 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err)
658 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag)
660 # Should also be able to retrieve them as columns from the catalog
661 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux)
662 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err)
664 # And we should be able to delete the slot, breaking the mapping.
665 table.schema.getAliasMap().erase(f"slot_{slotName}")
666 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid())
667 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey)
668 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey)
669 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey)
671 # When the slot has been deleted, attempting to access it should
672 # throw a LogicError.
673 with self.assertRaises(lsst.pex.exceptions.LogicError):
674 getattr(catalog, f"get{instFluxName}")()
675 with self.assertRaises(lsst.pex.exceptions.LogicError):
676 getattr(catalog, f"get{instFluxName}Err")()
678 def testFluxSlots(self):
679 """Check that all the expected flux slots are present & correct."""
680 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux",
681 "PsfFlux"]:
682 self._testFluxSlot(slotName)
684 # But, of course, we should not accept a slot which hasn't be defined.
685 with self.assertRaises(AttributeError):
686 self._testFluxSlot("NotExtantFlux")
688 def testStr(self):
689 """Check that the str() produced on a catalog contains expected things."""
690 string = str(self.catalog)
691 for field in ('id', 'coord_ra', 'coord_dec'):
692 self.assertIn(field, string)
694 def testRepr(self):
695 """Check that the repr() produced on a catalog contains expected things."""
696 string = repr(self.catalog)
697 self.assertIn(str(type(self.catalog)), string)
698 for field in ('id', 'coord_ra', 'coord_dec'):
699 self.assertIn(field, string)
701 def testStrNonContiguous(self):
702 """Check that str() doesn't fail on non-contiguous tables."""
703 del self.catalog[1]
704 string = str(self.catalog)
705 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string)
706 for field in ('id', 'coord_ra', 'coord_dec'):
707 self.assertIn(field, string)
709 def testRecordStr(self):
710 """Test that str(record) contains expected things."""
711 string = str(self.catalog[0])
712 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'):
713 self.assertIn(field, string)
715 def testRecordRepr(self):
716 """Test that repr(record) contains expected things."""
717 string = repr(self.catalog[0])
718 self.assertIn(str(type(self.catalog[0])), string)
719 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'):
720 self.assertIn(field, string)
722 def testGetNonContiguous(self):
723 """Check that we can index on non-contiguous tables"""
724 # Make a non-contiguous catalog
725 nonContiguous = type(self.catalog)(self.catalog.table)
726 for rr in reversed(self.catalog):
727 nonContiguous.append(rr)
728 num = len(self.catalog)
729 # Check assumptions
730 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog
731 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique
732 # Indexing with boolean array
733 select = np.zeros(num, dtype=bool)
734 select[1] = True
735 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"])
736 # Extracting a number column
737 column = "a_instFlux"
738 array = nonContiguous[column]
739 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column])
740 with self.assertRaises(ValueError):
741 array[1] = 1.2345 # Should be immutable
742 # Extracting a flag column
743 column = "a_flag"
744 array = nonContiguous[column]
745 np.testing.assert_equal(np.flip(array, 0), self.catalog[column])
746 with self.assertRaises(ValueError):
747 array[1] = True # Should be immutable
750class MemoryTester(lsst.utils.tests.MemoryTestCase):
751 pass
754def setup_module(module):
755 lsst.utils.tests.init()
758if __name__ == "__main__": 758 ↛ 759line 758 didn't jump to line 759, because the condition on line 758 was never true
759 lsst.utils.tests.init()
760 unittest.main()