Coverage for tests/test_sourceTable.py: 10%

509 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-03 02:47 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for table.SourceTable 

24 

25Run with: 

26 python test_sourceTable.py 

27or 

28 pytest test_sourceTable.py 

29""" 

30import os 

31import unittest 

32import tempfile 

33import pickle 

34import math 

35 

36import numpy as np 

37 

38import lsst.utils.tests 

39import lsst.pex.exceptions 

40import lsst.geom 

41import lsst.afw.table 

42import lsst.afw.geom 

43import lsst.afw.image 

44import lsst.afw.detection 

45 

46testPath = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49def makeArray(size, dtype): 

50 return np.array(np.random.randn(*size), dtype=dtype) 

51 

52 

53def makeCov(size, dtype): 

54 m = np.array(np.random.randn(size, size), dtype=dtype) 

55 return np.dot(m, m.transpose()) 

56 

57 

58def makeWcs(): 

59 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees, 

60 5.090329*lsst.geom.degrees) 

61 crpix = lsst.geom.Point2D(2036.0, 2000.0) 

62 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5]) 

63 cdMatrix.shape = (2, 2) 

64 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

65 

66 

67class SourceTableTestCase(lsst.utils.tests.TestCase): 

68 

69 def fillRecord(self, record): 

70 record.set(self.instFluxKey, np.random.randn()) 

71 record.set(self.instFluxErrKey, np.random.randn()) 

72 record.set(self.centroidKey.getX(), np.random.randn()) 

73 record.set(self.centroidKey.getY(), np.random.randn()) 

74 record.set(self.xErrKey, np.random.randn()) 

75 record.set(self.yErrKey, np.random.randn()) 

76 record.set(self.shapeKey.getIxx(), np.random.randn()) 

77 record.set(self.shapeKey.getIyy(), np.random.randn()) 

78 record.set(self.shapeKey.getIxy(), np.random.randn()) 

79 record.set(self.xxErrKey, np.random.randn()) 

80 record.set(self.yyErrKey, np.random.randn()) 

81 record.set(self.xyErrKey, np.random.randn()) 

82 record.set(self.psfShapeKey.getIxx(), np.random.randn()) 

83 record.set(self.psfShapeKey.getIyy(), np.random.randn()) 

84 record.set(self.psfShapeKey.getIxy(), np.random.randn()) 

85 record.set(self.fluxFlagKey, np.random.randn() > 0) 

86 record.set(self.centroidFlagKey, np.random.randn() > 0) 

87 record.set(self.shapeFlagKey, np.random.randn() > 0) 

88 

89 def setUp(self): 

90 np.random.seed(1) 

91 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

92 self.instFluxKey = self.schema.addField("a_instFlux", type="D") 

93 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D") 

94 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag") 

95 

96 # the meas field is added using a functor key, but the error is added 

97 # as scalars, as we lack a ResultKey functor as exists in meas_base 

98 self.centroidKey = lsst.afw.table.Point2DKey.addFields( 

99 self.schema, "b", "", "pixel") 

100 self.xErrKey = self.schema.addField("b_xErr", type="F") 

101 self.yErrKey = self.schema.addField("b_yErr", type="F") 

102 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag") 

103 

104 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

105 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL) 

106 self.xxErrKey = self.schema.addField("c_xxErr", type="F") 

107 self.xyErrKey = self.schema.addField("c_xyErr", type="F") 

108 self.yyErrKey = self.schema.addField("c_yyErr", type="F") 

109 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag") 

110 

111 self.psfShapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

112 self.schema, "d", "", lsst.afw.table.CoordinateType.PIXEL) 

113 self.psfShapeFlagKey = self.schema.addField("d_flag", type="Flag") 

114 

115 self.table = lsst.afw.table.SourceTable.make(self.schema) 

116 self.catalog = lsst.afw.table.SourceCatalog(self.table) 

117 self.record = self.catalog.addNew() 

118 self.fillRecord(self.record) 

119 self.record.setId(50) 

120 self.fillRecord(self.catalog.addNew()) 

121 self.fillRecord(self.catalog.addNew()) 

122 

123 def tearDown(self): 

124 del self.schema 

125 del self.record 

126 del self.table 

127 del self.catalog 

128 

129 def checkCanonical(self): 

130 self.assertEqual(self.record.get(self.instFluxKey), 

131 self.record.getPsfInstFlux()) 

132 self.assertEqual(self.record.get(self.fluxFlagKey), 

133 self.record.getPsfFluxFlag()) 

134 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

135 self.assertEqual(self.centroidKey.get(self.record), 

136 self.record.getCentroid()) 

137 self.assertFloatsAlmostEqual( 

138 math.fabs(self.record.get(self.xErrKey)), 

139 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

140 self.assertFloatsAlmostEqual( 

141 math.fabs(self.record.get(self.yErrKey)), 

142 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

143 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c") 

144 self.assertEqual(self.shapeKey.get(self.record), 

145 self.record.getShape()) 

146 self.assertFloatsAlmostEqual( 

147 math.fabs(self.record.get(self.xxErrKey)), 

148 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

149 self.assertFloatsAlmostEqual( 

150 math.fabs(self.record.get(self.yyErrKey)), 

151 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

152 self.assertFloatsAlmostEqual( 

153 math.fabs(self.record.get(self.xyErrKey)), 

154 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

155 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

156 self.assertEqual(self.psfShapeKey.get(self.record), 

157 self.record.getPsfShape()) 

158 

159 def testPersisted(self): 

160 self.table.definePsfFlux("a") 

161 self.table.defineCentroid("b") 

162 self.table.defineShape("c") 

163 self.table.definePsfShape("d") 

164 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

165 self.catalog.writeFits(filename) 

166 catalog = lsst.afw.table.SourceCatalog.readFits(filename) 

167 table = catalog.getTable() 

168 record = catalog[0] 

169 # I'm using the keys from the non-persisted table. They should work at least in the 

170 # current implementation 

171 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux()) 

172 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag()) 

173 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

174 centroid = self.centroidKey.get(self.record) 

175 self.assertEqual(centroid, record.getCentroid()) 

176 self.assertFloatsAlmostEqual( 

177 math.fabs(self.record.get(self.xErrKey)), 

178 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

179 self.assertFloatsAlmostEqual( 

180 math.fabs(self.record.get(self.yErrKey)), 

181 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

182 shape = self.shapeKey.get(self.record) 

183 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c") 

184 self.assertEqual(shape, record.getShape()) 

185 self.assertFloatsAlmostEqual( 

186 math.fabs(self.record.get(self.xxErrKey)), 

187 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

188 self.assertFloatsAlmostEqual( 

189 math.fabs(self.record.get(self.yyErrKey)), 

190 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

191 self.assertFloatsAlmostEqual( 

192 math.fabs(self.record.get(self.xyErrKey)), 

193 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

194 psfShape = self.psfShapeKey.get(self.record) 

195 self.assertEqual(table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

196 self.assertEqual(psfShape, record.getPsfShape()) 

197 

198 def testCanonical2(self): 

199 self.table.definePsfFlux("a") 

200 self.table.defineCentroid("b") 

201 self.table.defineShape("c") 

202 self.table.definePsfShape("d") 

203 self.checkCanonical() 

204 

205 def testPickle(self): 

206 p = pickle.dumps(self.catalog) 

207 new = pickle.loads(p) 

208 

209 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames()) 

210 self.assertEqual(len(self.catalog), len(new)) 

211 for r1, r2 in zip(self.catalog, new): 

212 # Columns that are easy to test 

213 for field in ("a_instFlux", "a_instFluxErr", "id"): 

214 k1 = self.catalog.schema.find(field).getKey() 

215 k2 = new.schema.find(field).getKey() 

216 self.assertEqual(r1[k1], r2[k2]) 

217 

218 def testCoordUpdate(self): 

219 self.table.defineCentroid("b") 

220 wcs = makeWcs() 

221 self.record.updateCoord(wcs) 

222 coord1 = self.record.getCoord() 

223 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey)) 

224 self.assertEqual(coord1, coord2) 

225 

226 def testSorting(self): 

227 self.assertFalse(self.catalog.isSorted()) 

228 self.catalog.sort() 

229 self.assertTrue(self.catalog.isSorted()) 

230 r = self.catalog.find(2) 

231 self.assertEqual(r["id"], 2) 

232 r = self.catalog.find(500) 

233 self.assertIsNone(r) 

234 

235 def testConversion(self): 

236 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog) 

237 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog) 

238 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True) 

239 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True) 

240 self.assertEqual(self.catalog.table, catalog1.table) 

241 self.assertEqual(self.catalog.table, catalog2.table) 

242 self.assertNotEqual(self.catalog.table, catalog3.table) 

243 self.assertNotEqual(self.catalog.table, catalog3.table) 

244 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4): 

245 self.assertEqual(r, r1) 

246 self.assertEqual(r, r2) 

247 self.assertNotEqual(r, r3) 

248 self.assertNotEqual(r, r4) 

249 self.assertEqual(r.getId(), r3.getId()) 

250 self.assertEqual(r.getId(), r4.getId()) 

251 

252 def testColumnView(self): 

253 cols1 = self.catalog.getColumnView() 

254 cols2 = self.catalog.columns 

255 self.assertIs(cols1, cols2) 

256 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView) 

257 self.table.definePsfFlux("a") 

258 self.table.defineCentroid("b") 

259 self.table.defineShape("c") 

260 self.table.definePsfShape("d") 

261 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux()) 

262 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr()) 

263 self.assertFloatsEqual(cols2["b_x"], cols2.getX()) 

264 self.assertFloatsEqual(cols2["b_y"], cols2.getY()) 

265 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx()) 

266 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy()) 

267 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy()) 

268 self.assertFloatsEqual(cols2["d_xx"], cols2.getPsfIxx()) 

269 self.assertFloatsEqual(cols2["d_yy"], cols2.getPsfIyy()) 

270 self.assertFloatsEqual(cols2["d_xy"], cols2.getPsfIxy()) 

271 

272 # Trying to access slots which have been removed should raise. 

273 self.catalog.table.schema.getAliasMap().erase("slot_Centroid") 

274 self.catalog.table.schema.getAliasMap().erase("slot_Shape") 

275 self.catalog.table.schema.getAliasMap().erase("slot_PsfShape") 

276 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy", "PsfIxx", "PsfIyy", "PsfIxy"]: 

277 with self.assertRaises(lsst.pex.exceptions.LogicError): 

278 getattr(self.catalog, f"get{quantity}")() 

279 

280 def testForwarding(self): 

281 """Verify that Catalog forwards unknown methods to its table and/or columns.""" 

282 self.table.definePsfFlux("a") 

283 self.table.defineCentroid("b") 

284 self.table.defineShape("c") 

285 self.assertFloatsEqual(self.catalog.columns["a_instFlux"], 

286 self.catalog["a_instFlux"]) 

287 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey], 

288 self.catalog.get(self.instFluxKey)) 

289 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey), 

290 self.catalog.getPsfInstFlux()) 

291 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey()) 

292 with self.assertRaises(AttributeError): 

293 self.catalog.foo() 

294 

295 def testBitsColumn(self): 

296 

297 allBits = self.catalog.getBits() 

298 someBits = self.catalog.getBits(["a_flag", "c_flag"]) 

299 self.assertEqual(allBits.getMask("a_flag"), 0x1) 

300 self.assertEqual(allBits.getMask("b_flag"), 0x2) 

301 self.assertEqual(allBits.getMask("c_flag"), 0x4) 

302 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1) 

303 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2) 

304 np.testing.assert_array_equal( 

305 (allBits.array & 0x1 != 0), self.catalog.columns["a_flag"]) 

306 np.testing.assert_array_equal( 

307 (allBits.array & 0x2 != 0), self.catalog.columns["b_flag"]) 

308 np.testing.assert_array_equal( 

309 (allBits.array & 0x4 != 0), self.catalog.columns["c_flag"]) 

310 np.testing.assert_array_equal( 

311 (someBits.array & 0x1 != 0), self.catalog.columns["a_flag"]) 

312 np.testing.assert_array_equal( 

313 (someBits.array & 0x2 != 0), self.catalog.columns["c_flag"]) 

314 

315 def testCast(self): 

316 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog) 

317 baseCat.cast(lsst.afw.table.SourceCatalog) 

318 

319 def testFootprints(self): 

320 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS 

321 ''' 

322 src1 = self.catalog.addNew() 

323 src2 = self.catalog.addNew() 

324 src3 = self.catalog.addNew() 

325 self.fillRecord(src1) 

326 self.fillRecord(src2) 

327 self.fillRecord(src3) 

328 src2.setParent(src1.getId()) 

329 

330 W, H = 100, 100 

331 mim = lsst.afw.image.MaskedImageF(W, H) 

332 im = mim.getImage() 

333 msk = mim.getMask() 

334 var = mim.getVariance() 

335 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int)) 

336 im.array[:] = y*1E6 + x*1E3 

337 msk.array[:] = (y << 8) | x 

338 var.array[:] = y*1E2 + x 

339 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50) 

340 circ = lsst.afw.detection.Footprint(spanSet) 

341 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim) 

342 src2.setFootprint(heavy) 

343 

344 for i, src in enumerate(self.catalog): 

345 if src != src2: 

346 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50) 

347 src.setFootprint(lsst.afw.detection.Footprint(spanSet)) 

348 

349 # insert this HeavyFootprint into an otherwise blank image (for comparing the results) 

350 mim2 = lsst.afw.image.MaskedImageF(W, H) 

351 heavy.insert(mim2) 

352 

353 with lsst.utils.tests.getTempFilePath(".fits") as fn: 

354 self.catalog.writeFits(fn) 

355 

356 cat2 = lsst.afw.table.SourceCatalog.readFits(fn) 

357 r2 = cat2[-2] 

358 h2 = r2.getFootprint() 

359 self.assertTrue(h2.isHeavy()) 

360 mim3 = lsst.afw.image.MaskedImageF(W, H) 

361 h2.insert(mim3) 

362 

363 self.assertFalse(cat2[-1].getFootprint().isHeavy()) 

364 self.assertFalse(cat2[-3].getFootprint().isHeavy()) 

365 self.assertFalse(cat2[0].getFootprint().isHeavy()) 

366 self.assertFalse(cat2[1].getFootprint().isHeavy()) 

367 self.assertFalse(cat2[2].getFootprint().isHeavy()) 

368 

369 if False: 

370 # Write out before-n-after FITS images 

371 for MI in [mim, mim2, mim3]: 

372 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits') 

373 os.close(f) 

374 MI.writeFits(fn2) 

375 print('wrote', fn2) 

376 

377 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray()) 

378 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray()) 

379 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray()) 

380 

381 im3 = mim3.getImage() 

382 ma3 = mim3.getMask() 

383 va3 = mim3.getVariance() 

384 for y in range(H): 

385 for x in range(W): 

386 if circ.contains(lsst.geom.Point2I(x, y)): 

387 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT]) 

388 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT]) 

389 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT]) 

390 else: 

391 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.) 

392 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.) 

393 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.) 

394 

395 cat3 = lsst.afw.table.SourceCatalog.readFits( 

396 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

397 for src in cat3: 

398 self.assertFalse(src.getFootprint().isHeavy()) 

399 cat4 = lsst.afw.table.SourceCatalog.readFits( 

400 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

401 for src in cat4: 

402 self.assertIsNone(src.getFootprint()) 

403 

404 self.catalog.writeFits( 

405 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

406 cat5 = lsst.afw.table.SourceCatalog.readFits(fn) 

407 for src in cat5: 

408 self.assertFalse(src.getFootprint().isHeavy()) 

409 

410 self.catalog.writeFits( 

411 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

412 cat6 = lsst.afw.table.SourceCatalog.readFits(fn) 

413 for src in cat6: 

414 self.assertIsNone(src.getFootprint()) 

415 

416 def testIdFactory(self): 

417 expId = int(1257198) 

418 reserved = 32 

419 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved) 

420 id1 = factory() 

421 id2 = factory() 

422 self.assertEqual(id2 - id1, 1) 

423 factory.notify(0xFFFFFFFF) 

424 with self.assertRaises(lsst.pex.exceptions.LengthError): 

425 factory() 

426 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

427 factory.notify(0x1FFFFFFFF) 

428 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

429 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved) 

430 

431 def testFamilies(self): 

432 self.catalog.sort() 

433 parents = self.catalog.getChildren(0) 

434 self.assertEqual(list(parents), list(self.catalog)) 

435 parentKey = lsst.afw.table.SourceTable.getParentKey() 

436 for parent in parents: 

437 self.assertEqual(parent.get(parentKey), 0) 

438 for i in range(10): 

439 child = self.catalog.addNew() 

440 self.fillRecord(child) 

441 child.set(parentKey, parent.getId()) 

442 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents], 

443 [record.getId() for record in self.catalog]) 

444 for parent, (children, ids) in zip(parents, childrenIter): 

445 self.assertEqual(len(children), 10) 

446 self.assertEqual(len(children), len(ids)) 

447 for child, id in zip(children, ids): 

448 self.assertEqual(child.getParent(), parent.getId()) 

449 self.assertEqual(child.getId(), id) 

450 

451 # Check detection of unsorted catalog 

452 self.catalog.sort(self.instFluxKey) 

453 with self.assertRaises(AssertionError): 

454 self.catalog.getChildren(0) 

455 self.catalog.sort(parentKey) 

456 self.catalog.getChildren(0) # Just care this succeeds 

457 

458 def testFitsReadVersion0Compatibility(self): 

459 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits")) 

460 self.assertTrue(cat.getPsfFluxSlot().isValid()) 

461 self.assertTrue(cat.getApFluxSlot().isValid()) 

462 self.assertTrue(cat.getGaussianFluxSlot().isValid()) 

463 self.assertTrue(cat.getModelFluxSlot().isValid()) 

464 self.assertTrue(cat.getCentroidSlot().isValid()) 

465 self.assertTrue(cat.getShapeSlot().isValid()) 

466 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(), 

467 cat.schema.find("flux_psf").key) 

468 self.assertEqual(cat.getApFluxSlot().getMeasKey(), 

469 cat.schema.find("flux_sinc").key) 

470 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(), 

471 cat.schema.find("flux_naive").key) 

472 self.assertEqual(cat.getModelFluxSlot().getMeasKey(), 

473 cat.schema.find("cmodel_flux").key) 

474 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(), 

475 cat.schema.find("centroid_sdss_x").key) 

476 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(), 

477 cat.schema.find("centroid_sdss_y").key) 

478 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(), 

479 cat.schema.find("shape_hsm_moments_xx").key) 

480 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(), 

481 cat.schema.find("shape_hsm_moments_yy").key) 

482 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(), 

483 cat.schema.find("shape_hsm_moments_xy").key) 

484 self.assertEqual(cat.getPsfFluxSlot().getErrKey(), 

485 cat.schema.find("flux_psf_err").key) 

486 self.assertEqual(cat.getApFluxSlot().getErrKey(), 

487 cat.schema.find("flux_sinc_err").key) 

488 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(), 

489 cat.schema.find("flux_naive_err").key) 

490 self.assertEqual(cat.getModelFluxSlot().getErrKey(), 

491 cat.schema.find("cmodel_flux_err").key) 

492 self.assertEqual( 

493 cat.getCentroidSlot().getErrKey(), 

494 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"])) 

495 self.assertEqual( 

496 cat.getShapeSlot().getErrKey(), 

497 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"])) 

498 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(), 

499 cat.schema.find("flux_psf_flags").key) 

500 self.assertEqual(cat.getApFluxSlot().getFlagKey(), 

501 cat.schema.find("flux_sinc_flags").key) 

502 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(), 

503 cat.schema.find("flux_naive_flags").key) 

504 self.assertEqual(cat.getModelFluxSlot().getFlagKey(), 

505 cat.schema.find("cmodel_flux_flags").key) 

506 self.assertEqual(cat.getCentroidSlot().getFlagKey(), 

507 cat.schema.find("centroid_sdss_flags").key) 

508 self.assertEqual(cat.getShapeSlot().getFlagKey(), 

509 cat.schema.find("shape_hsm_moments_flags").key) 

510 

511 def testFitsReadVersion1Compatibility(self): 

512 """Test reading of catalogs with version 1 schema 

513 

514 Version 1 catalogs need to have added aliases from Sigma->Err and 

515 from `_flux`->`_instFlux`. 

516 """ 

517 cat = lsst.afw.table.SourceCatalog.readFits( 

518 os.path.join(testPath, "data", "sourceTable-v1.fits")) 

519 self.assertEqual( 

520 cat.getCentroidSlot().getErrKey(), 

521 lsst.afw.table.CovarianceMatrix2fKey( 

522 cat.schema["slot_Centroid"], 

523 ["x", "y"])) 

524 self.assertEqual( 

525 cat.getShapeSlot().getErrKey(), 

526 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"])) 

527 # check the flux->instFlux conversion 

528 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

529 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

530 

531 def testFitsReadVersion2CompatibilityRealSourceCatalog(self): 

532 """DM-15891: some fields were getting aliases they shouldn't have.""" 

533 cat = lsst.afw.table.SourceCatalog.readFits( 

534 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits")) 

535 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema) 

536 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema) 

537 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema) 

538 

539 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self): 

540 """DM-16068: some fields were not getting aliases they should have 

541 

542 In particular, the alias setting relies on flux fields having their 

543 units set properly. Prior to a resolution of DM-16068, the units 

544 were not getting set for several CModel flux fields and one deblender 

545 field (deblend_psfFlux), and thus were not getting the 

546 `_flux`->`_instFlux` aliases set. 

547 

548 NOTE: this test will (is meant to) fail on the read until DM-16068 is 

549 resolved. The error is: 

550 

551 lsst::pex::exceptions::NotFoundError: 'Field or subfield with 

552 name 'modelfit_CModel_instFlux' not found with type 'D'.' 

553 """ 

554 cat = lsst.afw.table.SourceCatalog.readFits( 

555 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits")) 

556 self.assertIn('modelfit_CModel_instFlux', cat.schema) 

557 self.assertIn('modelfit_CModel_instFluxErr', cat.schema) 

558 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema) 

559 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema) 

560 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema) 

561 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema) 

562 

563 def testFitsVersion2Compatibility(self): 

564 """Test reading of catalogs with version 2 schema 

565 

566 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`. 

567 """ 

568 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits")) 

569 # check the flux->instFlux conversion 

570 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

571 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

572 

573 def testDM1083(self): 

574 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

575 st = lsst.afw.table.SourceTable.make(schema) 

576 cat = lsst.afw.table.SourceCatalog(st) 

577 tmp = lsst.afw.table.SourceCatalog(cat.getTable()) 

578 record = tmp.addNew() 

579 cat.extend(tmp) 

580 self.assertEqual(cat[0].getId(), record.getId()) 

581 # check that the same record is in both catalogs (not a copy) 

582 record.setId(15) 

583 self.assertEqual(cat[0].getId(), record.getId()) 

584 

585 def testSlotUndefine(self): 

586 """Test that we can correctly define and undefine a slot after a SourceTable has been created""" 

587 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

588 key = schema.addField("a_instFlux", type=np.float64, doc="flux field") 

589 table = lsst.afw.table.SourceTable.make(schema) 

590 table.definePsfFlux("a") 

591 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key) 

592 table.schema.getAliasMap().erase("slot_PsfFlux") 

593 self.assertFalse(table.getPsfFluxSlot().isValid()) 

594 

595 def testOldFootprintPersistence(self): 

596 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older 

597 version of the pipeline with a different format. 

598 """ 

599 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits") 

600 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename) 

601 self.assertEqual(len(catalog1), 2) 

602 with self.assertRaises(LookupError): 

603 catalog1.schema.find("footprint") 

604 fp1 = catalog1[0].getFootprint() 

605 fp2 = catalog1[1].getFootprint() 

606 self.assertEqual(fp1.getArea(), 495) 

607 self.assertEqual(fp2.getArea(), 767) 

608 self.assertFalse(fp1.isHeavy()) 

609 self.assertTrue(fp2.isHeavy()) 

610 self.assertEqual(len(fp1.getSpans()), 29) 

611 self.assertEqual(len(fp2.getSpans()), 44) 

612 self.assertEqual(len(fp1.getPeaks()), 1) 

613 self.assertEqual(len(fp2.getPeaks()), 1) 

614 self.assertEqual(fp1.getBBox(), 

615 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29))) 

616 self.assertEqual(fp2.getBBox(), 

617 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38))) 

618 hfp = lsst.afw.detection.HeavyFootprintF(fp2) 

619 self.assertEqual(len(hfp.getImageArray()), fp2.getArea()) 

620 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea()) 

621 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea()) 

622 catalog2 = lsst.afw.table.SourceCatalog.readFits( 

623 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

624 self.assertEqual(list(fp1.getSpans()), 

625 list(catalog2[0].getFootprint().getSpans())) 

626 self.assertEqual(list(fp2.getSpans()), 

627 list(catalog2[1].getFootprint().getSpans())) 

628 self.assertFalse(catalog2[1].getFootprint().isHeavy()) 

629 catalog3 = lsst.afw.table.SourceCatalog.readFits( 

630 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

631 self.assertEqual(catalog3[0].getFootprint(), None) 

632 self.assertEqual(catalog3[1].getFootprint(), None) 

633 

634 def _testFluxSlot(self, slotName): 

635 """Demonstrate that we can create & use the named Flux slot.""" 

636 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

637 baseName = "afw_Test" 

638 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux") 

639 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty") 

640 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag") 

641 catalog = lsst.afw.table.SourceCatalog(schema) 

642 table = catalog.table 

643 

644 # Initially, the slot is undefined. 

645 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

646 

647 # After definition, it maps to the keys defined above. 

648 getattr(table, f"define{slotName}")(baseName) 

649 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid()) 

650 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

651 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

652 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

653 

654 # We should be able to retrieve arbitrary values set in records. 

655 record = catalog.addNew() 

656 instFlux, err, flag = 10.0, 1.0, False 

657 record.set(instFluxKey, instFlux) 

658 record.set(errKey, err) 

659 record.set(flagKey, flag) 

660 instFluxName = slotName.replace("Flux", "InstFlux") 

661 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux) 

662 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err) 

663 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag) 

664 

665 # Should also be able to retrieve them as columns from the catalog 

666 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux) 

667 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err) 

668 

669 # And we should be able to delete the slot, breaking the mapping. 

670 table.schema.getAliasMap().erase(f"slot_{slotName}") 

671 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

672 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

673 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

674 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

675 

676 # When the slot has been deleted, attempting to access it should 

677 # throw a LogicError. 

678 with self.assertRaises(lsst.pex.exceptions.LogicError): 

679 getattr(catalog, f"get{instFluxName}")() 

680 with self.assertRaises(lsst.pex.exceptions.LogicError): 

681 getattr(catalog, f"get{instFluxName}Err")() 

682 

683 def testFluxSlots(self): 

684 """Check that all the expected flux slots are present & correct.""" 

685 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux", 

686 "PsfFlux"]: 

687 self._testFluxSlot(slotName) 

688 

689 # But, of course, we should not accept a slot which hasn't be defined. 

690 with self.assertRaises(AttributeError): 

691 self._testFluxSlot("NotExtantFlux") 

692 

693 def testStr(self): 

694 """Check that the str() produced on a catalog contains expected things.""" 

695 string = str(self.catalog) 

696 for field in ('id', 'coord_ra', 'coord_dec'): 

697 self.assertIn(field, string) 

698 

699 def testRepr(self): 

700 """Check that the repr() produced on a catalog contains expected things.""" 

701 string = repr(self.catalog) 

702 self.assertIn(str(type(self.catalog)), string) 

703 for field in ('id', 'coord_ra', 'coord_dec'): 

704 self.assertIn(field, string) 

705 

706 def testStrNonContiguous(self): 

707 """Check that str() doesn't fail on non-contiguous tables.""" 

708 del self.catalog[1] 

709 string = str(self.catalog) 

710 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string) 

711 for field in ('id', 'coord_ra', 'coord_dec'): 

712 self.assertIn(field, string) 

713 

714 def testRecordStr(self): 

715 """Test that str(record) contains expected things.""" 

716 string = str(self.catalog[0]) 

717 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

718 self.assertIn(field, string) 

719 

720 def testRecordRepr(self): 

721 """Test that repr(record) contains expected things.""" 

722 string = repr(self.catalog[0]) 

723 self.assertIn(str(type(self.catalog[0])), string) 

724 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

725 self.assertIn(field, string) 

726 

727 def testGetNonContiguous(self): 

728 """Check that we can index on non-contiguous tables""" 

729 # Make a non-contiguous catalog 

730 nonContiguous = type(self.catalog)(self.catalog.table) 

731 for rr in reversed(self.catalog): 

732 nonContiguous.append(rr) 

733 num = len(self.catalog) 

734 # Check assumptions 

735 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog 

736 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique 

737 # Indexing with boolean array 

738 select = np.zeros(num, dtype=bool) 

739 select[1] = True 

740 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"]) 

741 # Extracting a number column 

742 column = "a_instFlux" 

743 array = nonContiguous[column] 

744 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column]) 

745 with self.assertRaises(ValueError): 

746 array[1] = 1.2345 # Should be immutable 

747 # Extracting a flag column 

748 column = "a_flag" 

749 array = nonContiguous[column] 

750 np.testing.assert_equal(np.flip(array, 0), self.catalog[column]) 

751 with self.assertRaises(ValueError): 

752 array[1] = True # Should be immutable 

753 

754 

755class MemoryTester(lsst.utils.tests.MemoryTestCase): 

756 pass 

757 

758 

759def setup_module(module): 

760 lsst.utils.tests.init() 

761 

762 

763if __name__ == "__main__": 763 ↛ 764line 763 didn't jump to line 764, because the condition on line 763 was never true

764 lsst.utils.tests.init() 

765 unittest.main()