Coverage for tests/test_sourceTable.py: 10%

491 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 17:50 -0800

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for table.SourceTable 

24 

25Run with: 

26 python test_sourceTable.py 

27or 

28 pytest test_sourceTable.py 

29""" 

30import os 

31import unittest 

32import tempfile 

33import pickle 

34import math 

35 

36import numpy as np 

37 

38import lsst.utils.tests 

39import lsst.pex.exceptions 

40import lsst.geom 

41import lsst.afw.table 

42import lsst.afw.geom 

43import lsst.afw.image 

44import lsst.afw.detection 

45 

46testPath = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49def makeArray(size, dtype): 

50 return np.array(np.random.randn(*size), dtype=dtype) 

51 

52 

53def makeCov(size, dtype): 

54 m = np.array(np.random.randn(size, size), dtype=dtype) 

55 return np.dot(m, m.transpose()) 

56 

57 

58def makeWcs(): 

59 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees, 

60 5.090329*lsst.geom.degrees) 

61 crpix = lsst.geom.Point2D(2036.0, 2000.0) 

62 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5]) 

63 cdMatrix.shape = (2, 2) 

64 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

65 

66 

67class SourceTableTestCase(lsst.utils.tests.TestCase): 

68 

69 def fillRecord(self, record): 

70 record.set(self.instFluxKey, np.random.randn()) 

71 record.set(self.instFluxErrKey, np.random.randn()) 

72 record.set(self.centroidKey.getX(), np.random.randn()) 

73 record.set(self.centroidKey.getY(), np.random.randn()) 

74 record.set(self.xErrKey, np.random.randn()) 

75 record.set(self.yErrKey, np.random.randn()) 

76 record.set(self.shapeKey.getIxx(), np.random.randn()) 

77 record.set(self.shapeKey.getIyy(), np.random.randn()) 

78 record.set(self.shapeKey.getIxy(), np.random.randn()) 

79 record.set(self.xxErrKey, np.random.randn()) 

80 record.set(self.yyErrKey, np.random.randn()) 

81 record.set(self.xyErrKey, np.random.randn()) 

82 record.set(self.fluxFlagKey, np.random.randn() > 0) 

83 record.set(self.centroidFlagKey, np.random.randn() > 0) 

84 record.set(self.shapeFlagKey, np.random.randn() > 0) 

85 

86 def setUp(self): 

87 np.random.seed(1) 

88 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

89 self.instFluxKey = self.schema.addField("a_instFlux", type="D") 

90 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D") 

91 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag") 

92 

93 # the meas field is added using a functor key, but the error is added 

94 # as scalars, as we lack a ResultKey functor as exists in meas_base 

95 self.centroidKey = lsst.afw.table.Point2DKey.addFields( 

96 self.schema, "b", "", "pixel") 

97 self.xErrKey = self.schema.addField("b_xErr", type="F") 

98 self.yErrKey = self.schema.addField("b_yErr", type="F") 

99 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag") 

100 

101 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

102 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL) 

103 self.xxErrKey = self.schema.addField("c_xxErr", type="F") 

104 self.xyErrKey = self.schema.addField("c_xyErr", type="F") 

105 self.yyErrKey = self.schema.addField("c_yyErr", type="F") 

106 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag") 

107 

108 self.table = lsst.afw.table.SourceTable.make(self.schema) 

109 self.catalog = lsst.afw.table.SourceCatalog(self.table) 

110 self.record = self.catalog.addNew() 

111 self.fillRecord(self.record) 

112 self.record.setId(50) 

113 self.fillRecord(self.catalog.addNew()) 

114 self.fillRecord(self.catalog.addNew()) 

115 

116 def tearDown(self): 

117 del self.schema 

118 del self.record 

119 del self.table 

120 del self.catalog 

121 

122 def checkCanonical(self): 

123 self.assertEqual(self.record.get(self.instFluxKey), 

124 self.record.getPsfInstFlux()) 

125 self.assertEqual(self.record.get(self.fluxFlagKey), 

126 self.record.getPsfFluxFlag()) 

127 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

128 self.assertEqual(self.centroidKey.get(self.record), 

129 self.record.getCentroid()) 

130 self.assertFloatsAlmostEqual( 

131 math.fabs(self.record.get(self.xErrKey)), 

132 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

133 self.assertFloatsAlmostEqual( 

134 math.fabs(self.record.get(self.yErrKey)), 

135 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

136 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c") 

137 self.assertEqual(self.shapeKey.get(self.record), 

138 self.record.getShape()) 

139 self.assertFloatsAlmostEqual( 

140 math.fabs(self.record.get(self.xxErrKey)), 

141 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

142 self.assertFloatsAlmostEqual( 

143 math.fabs(self.record.get(self.yyErrKey)), 

144 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

145 self.assertFloatsAlmostEqual( 

146 math.fabs(self.record.get(self.xyErrKey)), 

147 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

148 

149 def testPersisted(self): 

150 self.table.definePsfFlux("a") 

151 self.table.defineCentroid("b") 

152 self.table.defineShape("c") 

153 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

154 self.catalog.writeFits(filename) 

155 catalog = lsst.afw.table.SourceCatalog.readFits(filename) 

156 table = catalog.getTable() 

157 record = catalog[0] 

158 # I'm using the keys from the non-persisted table. They should work at least in the 

159 # current implementation 

160 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux()) 

161 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag()) 

162 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

163 centroid = self.centroidKey.get(self.record) 

164 self.assertEqual(centroid, record.getCentroid()) 

165 self.assertFloatsAlmostEqual( 

166 math.fabs(self.record.get(self.xErrKey)), 

167 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

168 self.assertFloatsAlmostEqual( 

169 math.fabs(self.record.get(self.yErrKey)), 

170 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

171 shape = self.shapeKey.get(self.record) 

172 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c") 

173 self.assertEqual(shape, record.getShape()) 

174 self.assertFloatsAlmostEqual( 

175 math.fabs(self.record.get(self.xxErrKey)), 

176 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

177 self.assertFloatsAlmostEqual( 

178 math.fabs(self.record.get(self.yyErrKey)), 

179 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

180 self.assertFloatsAlmostEqual( 

181 math.fabs(self.record.get(self.xyErrKey)), 

182 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

183 

184 def testCanonical2(self): 

185 self.table.definePsfFlux("a") 

186 self.table.defineCentroid("b") 

187 self.table.defineShape("c") 

188 self.checkCanonical() 

189 

190 def testPickle(self): 

191 p = pickle.dumps(self.catalog) 

192 new = pickle.loads(p) 

193 

194 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames()) 

195 self.assertEqual(len(self.catalog), len(new)) 

196 for r1, r2 in zip(self.catalog, new): 

197 # Columns that are easy to test 

198 for field in ("a_instFlux", "a_instFluxErr", "id"): 

199 k1 = self.catalog.schema.find(field).getKey() 

200 k2 = new.schema.find(field).getKey() 

201 self.assertEqual(r1[k1], r2[k2]) 

202 

203 def testCoordUpdate(self): 

204 self.table.defineCentroid("b") 

205 wcs = makeWcs() 

206 self.record.updateCoord(wcs) 

207 coord1 = self.record.getCoord() 

208 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey)) 

209 self.assertEqual(coord1, coord2) 

210 

211 def testSorting(self): 

212 self.assertFalse(self.catalog.isSorted()) 

213 self.catalog.sort() 

214 self.assertTrue(self.catalog.isSorted()) 

215 r = self.catalog.find(2) 

216 self.assertEqual(r["id"], 2) 

217 r = self.catalog.find(500) 

218 self.assertIsNone(r) 

219 

220 def testConversion(self): 

221 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog) 

222 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog) 

223 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True) 

224 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True) 

225 self.assertEqual(self.catalog.table, catalog1.table) 

226 self.assertEqual(self.catalog.table, catalog2.table) 

227 self.assertNotEqual(self.catalog.table, catalog3.table) 

228 self.assertNotEqual(self.catalog.table, catalog3.table) 

229 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4): 

230 self.assertEqual(r, r1) 

231 self.assertEqual(r, r2) 

232 self.assertNotEqual(r, r3) 

233 self.assertNotEqual(r, r4) 

234 self.assertEqual(r.getId(), r3.getId()) 

235 self.assertEqual(r.getId(), r4.getId()) 

236 

237 def testColumnView(self): 

238 cols1 = self.catalog.getColumnView() 

239 cols2 = self.catalog.columns 

240 self.assertIs(cols1, cols2) 

241 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView) 

242 self.table.definePsfFlux("a") 

243 self.table.defineCentroid("b") 

244 self.table.defineShape("c") 

245 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux()) 

246 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr()) 

247 self.assertFloatsEqual(cols2["b_x"], cols2.getX()) 

248 self.assertFloatsEqual(cols2["b_y"], cols2.getY()) 

249 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx()) 

250 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy()) 

251 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy()) 

252 

253 # Trying to access slots which have been removed should raise. 

254 self.catalog.table.schema.getAliasMap().erase("slot_Centroid") 

255 self.catalog.table.schema.getAliasMap().erase("slot_Shape") 

256 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy"]: 

257 with self.assertRaises(lsst.pex.exceptions.LogicError): 

258 getattr(self.catalog, f"get{quantity}")() 

259 

260 def testForwarding(self): 

261 """Verify that Catalog forwards unknown methods to its table and/or columns.""" 

262 self.table.definePsfFlux("a") 

263 self.table.defineCentroid("b") 

264 self.table.defineShape("c") 

265 self.assertFloatsEqual(self.catalog.columns["a_instFlux"], 

266 self.catalog["a_instFlux"]) 

267 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey], 

268 self.catalog.get(self.instFluxKey)) 

269 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey), 

270 self.catalog.getPsfInstFlux()) 

271 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey()) 

272 with self.assertRaises(AttributeError): 

273 self.catalog.foo() 

274 

275 def testBitsColumn(self): 

276 

277 allBits = self.catalog.getBits() 

278 someBits = self.catalog.getBits(["a_flag", "c_flag"]) 

279 self.assertEqual(allBits.getMask("a_flag"), 0x1) 

280 self.assertEqual(allBits.getMask("b_flag"), 0x2) 

281 self.assertEqual(allBits.getMask("c_flag"), 0x4) 

282 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1) 

283 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2) 

284 np.testing.assert_array_equal( 

285 (allBits.array & 0x1 != 0), self.catalog.columns["a_flag"]) 

286 np.testing.assert_array_equal( 

287 (allBits.array & 0x2 != 0), self.catalog.columns["b_flag"]) 

288 np.testing.assert_array_equal( 

289 (allBits.array & 0x4 != 0), self.catalog.columns["c_flag"]) 

290 np.testing.assert_array_equal( 

291 (someBits.array & 0x1 != 0), self.catalog.columns["a_flag"]) 

292 np.testing.assert_array_equal( 

293 (someBits.array & 0x2 != 0), self.catalog.columns["c_flag"]) 

294 

295 def testCast(self): 

296 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog) 

297 baseCat.cast(lsst.afw.table.SourceCatalog) 

298 

299 def testFootprints(self): 

300 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS 

301 ''' 

302 src1 = self.catalog.addNew() 

303 src2 = self.catalog.addNew() 

304 src3 = self.catalog.addNew() 

305 self.fillRecord(src1) 

306 self.fillRecord(src2) 

307 self.fillRecord(src3) 

308 src2.setParent(src1.getId()) 

309 

310 W, H = 100, 100 

311 mim = lsst.afw.image.MaskedImageF(W, H) 

312 im = mim.getImage() 

313 msk = mim.getMask() 

314 var = mim.getVariance() 

315 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int)) 

316 im.array[:] = y*1E6 + x*1E3 

317 msk.array[:] = (y << 8) | x 

318 var.array[:] = y*1E2 + x 

319 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50) 

320 circ = lsst.afw.detection.Footprint(spanSet) 

321 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim) 

322 src2.setFootprint(heavy) 

323 

324 for i, src in enumerate(self.catalog): 

325 if src != src2: 

326 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50) 

327 src.setFootprint(lsst.afw.detection.Footprint(spanSet)) 

328 

329 # insert this HeavyFootprint into an otherwise blank image (for comparing the results) 

330 mim2 = lsst.afw.image.MaskedImageF(W, H) 

331 heavy.insert(mim2) 

332 

333 with lsst.utils.tests.getTempFilePath(".fits") as fn: 

334 self.catalog.writeFits(fn) 

335 

336 cat2 = lsst.afw.table.SourceCatalog.readFits(fn) 

337 r2 = cat2[-2] 

338 h2 = r2.getFootprint() 

339 self.assertTrue(h2.isHeavy()) 

340 mim3 = lsst.afw.image.MaskedImageF(W, H) 

341 h2.insert(mim3) 

342 

343 self.assertFalse(cat2[-1].getFootprint().isHeavy()) 

344 self.assertFalse(cat2[-3].getFootprint().isHeavy()) 

345 self.assertFalse(cat2[0].getFootprint().isHeavy()) 

346 self.assertFalse(cat2[1].getFootprint().isHeavy()) 

347 self.assertFalse(cat2[2].getFootprint().isHeavy()) 

348 

349 if False: 

350 # Write out before-n-after FITS images 

351 for MI in [mim, mim2, mim3]: 

352 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits') 

353 os.close(f) 

354 MI.writeFits(fn2) 

355 print('wrote', fn2) 

356 

357 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray()) 

358 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray()) 

359 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray()) 

360 

361 im3 = mim3.getImage() 

362 ma3 = mim3.getMask() 

363 va3 = mim3.getVariance() 

364 for y in range(H): 

365 for x in range(W): 

366 if circ.contains(lsst.geom.Point2I(x, y)): 

367 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT]) 

368 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT]) 

369 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT]) 

370 else: 

371 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.) 

372 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.) 

373 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.) 

374 

375 cat3 = lsst.afw.table.SourceCatalog.readFits( 

376 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

377 for src in cat3: 

378 self.assertFalse(src.getFootprint().isHeavy()) 

379 cat4 = lsst.afw.table.SourceCatalog.readFits( 

380 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

381 for src in cat4: 

382 self.assertIsNone(src.getFootprint()) 

383 

384 self.catalog.writeFits( 

385 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

386 cat5 = lsst.afw.table.SourceCatalog.readFits(fn) 

387 for src in cat5: 

388 self.assertFalse(src.getFootprint().isHeavy()) 

389 

390 self.catalog.writeFits( 

391 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

392 cat6 = lsst.afw.table.SourceCatalog.readFits(fn) 

393 for src in cat6: 

394 self.assertIsNone(src.getFootprint()) 

395 

396 def testIdFactory(self): 

397 expId = int(1257198) 

398 reserved = 32 

399 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved) 

400 id1 = factory() 

401 id2 = factory() 

402 self.assertEqual(id2 - id1, 1) 

403 factory.notify(0xFFFFFFFF) 

404 with self.assertRaises(lsst.pex.exceptions.LengthError): 

405 factory() 

406 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

407 factory.notify(0x1FFFFFFFF) 

408 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

409 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved) 

410 

411 def testFamilies(self): 

412 self.catalog.sort() 

413 parents = self.catalog.getChildren(0) 

414 self.assertEqual(list(parents), list(self.catalog)) 

415 parentKey = lsst.afw.table.SourceTable.getParentKey() 

416 for parent in parents: 

417 self.assertEqual(parent.get(parentKey), 0) 

418 for i in range(10): 

419 child = self.catalog.addNew() 

420 self.fillRecord(child) 

421 child.set(parentKey, parent.getId()) 

422 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents], 

423 [record.getId() for record in self.catalog]) 

424 for parent, (children, ids) in zip(parents, childrenIter): 

425 self.assertEqual(len(children), 10) 

426 self.assertEqual(len(children), len(ids)) 

427 for child, id in zip(children, ids): 

428 self.assertEqual(child.getParent(), parent.getId()) 

429 self.assertEqual(child.getId(), id) 

430 

431 # Check detection of unsorted catalog 

432 self.catalog.sort(self.instFluxKey) 

433 with self.assertRaises(AssertionError): 

434 self.catalog.getChildren(0) 

435 self.catalog.sort(parentKey) 

436 self.catalog.getChildren(0) # Just care this succeeds 

437 

438 def testFitsReadVersion0Compatibility(self): 

439 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits")) 

440 self.assertTrue(cat.getPsfFluxSlot().isValid()) 

441 self.assertTrue(cat.getApFluxSlot().isValid()) 

442 self.assertTrue(cat.getGaussianFluxSlot().isValid()) 

443 self.assertTrue(cat.getModelFluxSlot().isValid()) 

444 self.assertTrue(cat.getCentroidSlot().isValid()) 

445 self.assertTrue(cat.getShapeSlot().isValid()) 

446 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(), 

447 cat.schema.find("flux_psf").key) 

448 self.assertEqual(cat.getApFluxSlot().getMeasKey(), 

449 cat.schema.find("flux_sinc").key) 

450 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(), 

451 cat.schema.find("flux_naive").key) 

452 self.assertEqual(cat.getModelFluxSlot().getMeasKey(), 

453 cat.schema.find("cmodel_flux").key) 

454 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(), 

455 cat.schema.find("centroid_sdss_x").key) 

456 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(), 

457 cat.schema.find("centroid_sdss_y").key) 

458 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(), 

459 cat.schema.find("shape_hsm_moments_xx").key) 

460 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(), 

461 cat.schema.find("shape_hsm_moments_yy").key) 

462 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(), 

463 cat.schema.find("shape_hsm_moments_xy").key) 

464 self.assertEqual(cat.getPsfFluxSlot().getErrKey(), 

465 cat.schema.find("flux_psf_err").key) 

466 self.assertEqual(cat.getApFluxSlot().getErrKey(), 

467 cat.schema.find("flux_sinc_err").key) 

468 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(), 

469 cat.schema.find("flux_naive_err").key) 

470 self.assertEqual(cat.getModelFluxSlot().getErrKey(), 

471 cat.schema.find("cmodel_flux_err").key) 

472 self.assertEqual( 

473 cat.getCentroidSlot().getErrKey(), 

474 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"])) 

475 self.assertEqual( 

476 cat.getShapeSlot().getErrKey(), 

477 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"])) 

478 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(), 

479 cat.schema.find("flux_psf_flags").key) 

480 self.assertEqual(cat.getApFluxSlot().getFlagKey(), 

481 cat.schema.find("flux_sinc_flags").key) 

482 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(), 

483 cat.schema.find("flux_naive_flags").key) 

484 self.assertEqual(cat.getModelFluxSlot().getFlagKey(), 

485 cat.schema.find("cmodel_flux_flags").key) 

486 self.assertEqual(cat.getCentroidSlot().getFlagKey(), 

487 cat.schema.find("centroid_sdss_flags").key) 

488 self.assertEqual(cat.getShapeSlot().getFlagKey(), 

489 cat.schema.find("shape_hsm_moments_flags").key) 

490 

491 def testFitsReadVersion1Compatibility(self): 

492 """Test reading of catalogs with version 1 schema 

493 

494 Version 1 catalogs need to have added aliases from Sigma->Err and 

495 from `_flux`->`_instFlux`. 

496 """ 

497 cat = lsst.afw.table.SourceCatalog.readFits( 

498 os.path.join(testPath, "data", "sourceTable-v1.fits")) 

499 self.assertEqual( 

500 cat.getCentroidSlot().getErrKey(), 

501 lsst.afw.table.CovarianceMatrix2fKey( 

502 cat.schema["slot_Centroid"], 

503 ["x", "y"])) 

504 self.assertEqual( 

505 cat.getShapeSlot().getErrKey(), 

506 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"])) 

507 # check the flux->instFlux conversion 

508 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

509 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

510 

511 def testFitsReadVersion2CompatibilityRealSourceCatalog(self): 

512 """DM-15891: some fields were getting aliases they shouldn't have.""" 

513 cat = lsst.afw.table.SourceCatalog.readFits( 

514 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits")) 

515 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema) 

516 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema) 

517 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema) 

518 

519 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self): 

520 """DM-16068: some fields were not getting aliases they should have 

521 

522 In particular, the alias setting relies on flux fields having their 

523 units set properly. Prior to a resolution of DM-16068, the units 

524 were not getting set for several CModel flux fields and one deblender 

525 field (deblend_psfFlux), and thus were not getting the 

526 `_flux`->`_instFlux` aliases set. 

527 

528 NOTE: this test will (is meant to) fail on the read until DM-16068 is 

529 resolved. The error is: 

530 

531 lsst::pex::exceptions::NotFoundError: 'Field or subfield with 

532 name 'modelfit_CModel_instFlux' not found with type 'D'.' 

533 """ 

534 cat = lsst.afw.table.SourceCatalog.readFits( 

535 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits")) 

536 self.assertIn('modelfit_CModel_instFlux', cat.schema) 

537 self.assertIn('modelfit_CModel_instFluxErr', cat.schema) 

538 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema) 

539 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema) 

540 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema) 

541 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema) 

542 

543 def testFitsVersion2Compatibility(self): 

544 """Test reading of catalogs with version 2 schema 

545 

546 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`. 

547 """ 

548 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits")) 

549 # check the flux->instFlux conversion 

550 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

551 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

552 

553 def testDM1083(self): 

554 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

555 st = lsst.afw.table.SourceTable.make(schema) 

556 cat = lsst.afw.table.SourceCatalog(st) 

557 tmp = lsst.afw.table.SourceCatalog(cat.getTable()) 

558 record = tmp.addNew() 

559 cat.extend(tmp) 

560 self.assertEqual(cat[0].getId(), record.getId()) 

561 # check that the same record is in both catalogs (not a copy) 

562 record.setId(15) 

563 self.assertEqual(cat[0].getId(), record.getId()) 

564 

565 def testSlotUndefine(self): 

566 """Test that we can correctly define and undefine a slot after a SourceTable has been created""" 

567 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

568 key = schema.addField("a_instFlux", type=np.float64, doc="flux field") 

569 table = lsst.afw.table.SourceTable.make(schema) 

570 table.definePsfFlux("a") 

571 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key) 

572 table.schema.getAliasMap().erase("slot_PsfFlux") 

573 self.assertFalse(table.getPsfFluxSlot().isValid()) 

574 

575 def testOldFootprintPersistence(self): 

576 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older 

577 version of the pipeline with a different format. 

578 """ 

579 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits") 

580 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename) 

581 self.assertEqual(len(catalog1), 2) 

582 with self.assertRaises(LookupError): 

583 catalog1.schema.find("footprint") 

584 fp1 = catalog1[0].getFootprint() 

585 fp2 = catalog1[1].getFootprint() 

586 self.assertEqual(fp1.getArea(), 495) 

587 self.assertEqual(fp2.getArea(), 767) 

588 self.assertFalse(fp1.isHeavy()) 

589 self.assertTrue(fp2.isHeavy()) 

590 self.assertEqual(len(fp1.getSpans()), 29) 

591 self.assertEqual(len(fp2.getSpans()), 44) 

592 self.assertEqual(len(fp1.getPeaks()), 1) 

593 self.assertEqual(len(fp2.getPeaks()), 1) 

594 self.assertEqual(fp1.getBBox(), 

595 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29))) 

596 self.assertEqual(fp2.getBBox(), 

597 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38))) 

598 hfp = lsst.afw.detection.HeavyFootprintF(fp2) 

599 self.assertEqual(len(hfp.getImageArray()), fp2.getArea()) 

600 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea()) 

601 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea()) 

602 catalog2 = lsst.afw.table.SourceCatalog.readFits( 

603 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

604 self.assertEqual(list(fp1.getSpans()), 

605 list(catalog2[0].getFootprint().getSpans())) 

606 self.assertEqual(list(fp2.getSpans()), 

607 list(catalog2[1].getFootprint().getSpans())) 

608 self.assertFalse(catalog2[1].getFootprint().isHeavy()) 

609 catalog3 = lsst.afw.table.SourceCatalog.readFits( 

610 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

611 self.assertEqual(catalog3[0].getFootprint(), None) 

612 self.assertEqual(catalog3[1].getFootprint(), None) 

613 

614 def _testFluxSlot(self, slotName): 

615 """Demonstrate that we can create & use the named Flux slot.""" 

616 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

617 baseName = "afw_Test" 

618 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux") 

619 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty") 

620 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag") 

621 catalog = lsst.afw.table.SourceCatalog(schema) 

622 table = catalog.table 

623 

624 # Initially, the slot is undefined. 

625 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

626 

627 # After definition, it maps to the keys defined above. 

628 getattr(table, f"define{slotName}")(baseName) 

629 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid()) 

630 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

631 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

632 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

633 

634 # We should be able to retrieve arbitrary values set in records. 

635 record = catalog.addNew() 

636 instFlux, err, flag = 10.0, 1.0, False 

637 record.set(instFluxKey, instFlux) 

638 record.set(errKey, err) 

639 record.set(flagKey, flag) 

640 instFluxName = slotName.replace("Flux", "InstFlux") 

641 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux) 

642 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err) 

643 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag) 

644 

645 # Should also be able to retrieve them as columns from the catalog 

646 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux) 

647 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err) 

648 

649 # And we should be able to delete the slot, breaking the mapping. 

650 table.schema.getAliasMap().erase(f"slot_{slotName}") 

651 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

652 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

653 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

654 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

655 

656 # When the slot has been deleted, attempting to access it should 

657 # throw a LogicError. 

658 with self.assertRaises(lsst.pex.exceptions.LogicError): 

659 getattr(catalog, f"get{instFluxName}")() 

660 with self.assertRaises(lsst.pex.exceptions.LogicError): 

661 getattr(catalog, f"get{instFluxName}Err")() 

662 

663 def testFluxSlots(self): 

664 """Check that all the expected flux slots are present & correct.""" 

665 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux", 

666 "PsfFlux"]: 

667 self._testFluxSlot(slotName) 

668 

669 # But, of course, we should not accept a slot which hasn't be defined. 

670 with self.assertRaises(AttributeError): 

671 self._testFluxSlot("NotExtantFlux") 

672 

673 def testStr(self): 

674 """Check that the str() produced on a catalog contains expected things.""" 

675 string = str(self.catalog) 

676 for field in ('id', 'coord_ra', 'coord_dec'): 

677 self.assertIn(field, string) 

678 

679 def testRepr(self): 

680 """Check that the repr() produced on a catalog contains expected things.""" 

681 string = repr(self.catalog) 

682 self.assertIn(str(type(self.catalog)), string) 

683 for field in ('id', 'coord_ra', 'coord_dec'): 

684 self.assertIn(field, string) 

685 

686 def testStrNonContiguous(self): 

687 """Check that str() doesn't fail on non-contiguous tables.""" 

688 del self.catalog[1] 

689 string = str(self.catalog) 

690 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string) 

691 for field in ('id', 'coord_ra', 'coord_dec'): 

692 self.assertIn(field, string) 

693 

694 def testRecordStr(self): 

695 """Test that str(record) contains expected things.""" 

696 string = str(self.catalog[0]) 

697 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

698 self.assertIn(field, string) 

699 

700 def testRecordRepr(self): 

701 """Test that repr(record) contains expected things.""" 

702 string = repr(self.catalog[0]) 

703 self.assertIn(str(type(self.catalog[0])), string) 

704 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

705 self.assertIn(field, string) 

706 

707 def testGetNonContiguous(self): 

708 """Check that we can index on non-contiguous tables""" 

709 # Make a non-contiguous catalog 

710 nonContiguous = type(self.catalog)(self.catalog.table) 

711 for rr in reversed(self.catalog): 

712 nonContiguous.append(rr) 

713 num = len(self.catalog) 

714 # Check assumptions 

715 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog 

716 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique 

717 # Indexing with boolean array 

718 select = np.zeros(num, dtype=bool) 

719 select[1] = True 

720 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"]) 

721 # Extracting a number column 

722 column = "a_instFlux" 

723 array = nonContiguous[column] 

724 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column]) 

725 with self.assertRaises(ValueError): 

726 array[1] = 1.2345 # Should be immutable 

727 # Extracting a flag column 

728 column = "a_flag" 

729 array = nonContiguous[column] 

730 np.testing.assert_equal(np.flip(array, 0), self.catalog[column]) 

731 with self.assertRaises(ValueError): 

732 array[1] = True # Should be immutable 

733 

734 

735class MemoryTester(lsst.utils.tests.MemoryTestCase): 

736 pass 

737 

738 

739def setup_module(module): 

740 lsst.utils.tests.init() 

741 

742 

743if __name__ == "__main__": 743 ↛ 744line 743 didn't jump to line 744, because the condition on line 743 was never true

744 lsst.utils.tests.init() 

745 unittest.main()