Coverage for tests/test_sourceTable.py: 10%

509 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-08 03:13 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for table.SourceTable 

24 

25Run with: 

26 python test_sourceTable.py 

27or 

28 pytest test_sourceTable.py 

29""" 

30import os 

31import unittest 

32import tempfile 

33import pickle 

34import math 

35 

36import numpy as np 

37 

38import lsst.utils.tests 

39import lsst.pex.exceptions 

40import lsst.geom 

41import lsst.afw.table 

42import lsst.afw.geom 

43import lsst.afw.image 

44import lsst.afw.detection 

45 

46testPath = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49def makeArray(size, dtype): 

50 return np.array(np.random.randn(*size), dtype=dtype) 

51 

52 

53def makeCov(size, dtype): 

54 m = np.array(np.random.randn(size, size), dtype=dtype) 

55 return np.dot(m, m.transpose()) 

56 

57 

58def makeWcs(): 

59 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees, 

60 5.090329*lsst.geom.degrees) 

61 crpix = lsst.geom.Point2D(2036.0, 2000.0) 

62 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5]) 

63 cdMatrix.shape = (2, 2) 

64 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

65 

66 

67class SourceTableTestCase(lsst.utils.tests.TestCase): 

68 

69 def fillRecord(self, record): 

70 record.set(self.instFluxKey, np.random.randn()) 

71 record.set(self.instFluxErrKey, np.random.randn()) 

72 record.set(self.centroidKey.getX(), np.random.randn()) 

73 record.set(self.centroidKey.getY(), np.random.randn()) 

74 record.set(self.xErrKey, np.random.randn()) 

75 record.set(self.yErrKey, np.random.randn()) 

76 record.set(self.shapeKey.getIxx(), np.random.randn()) 

77 record.set(self.shapeKey.getIyy(), np.random.randn()) 

78 record.set(self.shapeKey.getIxy(), np.random.randn()) 

79 record.set(self.xxErrKey, np.random.randn()) 

80 record.set(self.yyErrKey, np.random.randn()) 

81 record.set(self.xyErrKey, np.random.randn()) 

82 record.set(self.psfShapeKey.getIxx(), np.random.randn()) 

83 record.set(self.psfShapeKey.getIyy(), np.random.randn()) 

84 record.set(self.psfShapeKey.getIxy(), np.random.randn()) 

85 record.set(self.fluxFlagKey, np.random.randn() > 0) 

86 record.set(self.centroidFlagKey, np.random.randn() > 0) 

87 record.set(self.shapeFlagKey, np.random.randn() > 0) 

88 

89 def setUp(self): 

90 np.random.seed(1) 

91 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

92 self.instFluxKey = self.schema.addField("a_instFlux", type="D") 

93 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D") 

94 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag") 

95 

96 # the meas field is added using a functor key, but the error is added 

97 # as scalars, as we lack a ResultKey functor as exists in meas_base 

98 self.centroidKey = lsst.afw.table.Point2DKey.addFields( 

99 self.schema, "b", "", "pixel") 

100 self.xErrKey = self.schema.addField("b_xErr", type="F") 

101 self.yErrKey = self.schema.addField("b_yErr", type="F") 

102 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag") 

103 

104 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

105 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL) 

106 self.xxErrKey = self.schema.addField("c_xxErr", type="F") 

107 self.xyErrKey = self.schema.addField("c_xyErr", type="F") 

108 self.yyErrKey = self.schema.addField("c_yyErr", type="F") 

109 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag") 

110 

111 self.psfShapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

112 self.schema, "d", "", lsst.afw.table.CoordinateType.PIXEL) 

113 self.psfShapeFlagKey = self.schema.addField("d_flag", type="Flag") 

114 

115 self.table = lsst.afw.table.SourceTable.make(self.schema) 

116 self.catalog = lsst.afw.table.SourceCatalog(self.table) 

117 self.record = self.catalog.addNew() 

118 self.fillRecord(self.record) 

119 self.record.setId(50) 

120 self.fillRecord(self.catalog.addNew()) 

121 self.fillRecord(self.catalog.addNew()) 

122 

123 def tearDown(self): 

124 del self.schema 

125 del self.record 

126 del self.table 

127 del self.catalog 

128 

129 def checkCanonical(self): 

130 self.assertEqual(self.record.get(self.instFluxKey), 

131 self.record.getPsfInstFlux()) 

132 self.assertEqual(self.record.get(self.fluxFlagKey), 

133 self.record.getPsfFluxFlag()) 

134 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

135 self.assertEqual(self.centroidKey.get(self.record), 

136 self.record.getCentroid()) 

137 self.assertFloatsAlmostEqual( 

138 math.fabs(self.record.get(self.xErrKey)), 

139 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

140 self.assertFloatsAlmostEqual( 

141 math.fabs(self.record.get(self.yErrKey)), 

142 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

143 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c") 

144 self.assertEqual(self.shapeKey.get(self.record), 

145 self.record.getShape()) 

146 self.assertFloatsAlmostEqual( 

147 math.fabs(self.record.get(self.xxErrKey)), 

148 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

149 self.assertFloatsAlmostEqual( 

150 math.fabs(self.record.get(self.yyErrKey)), 

151 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

152 self.assertFloatsAlmostEqual( 

153 math.fabs(self.record.get(self.xyErrKey)), 

154 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

155 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

156 self.assertEqual(self.psfShapeKey.get(self.record), 

157 self.record.getPsfShape()) 

158 

159 def testPersisted(self): 

160 self.table.definePsfFlux("a") 

161 self.table.defineCentroid("b") 

162 self.table.defineShape("c") 

163 self.table.definePsfShape("d") 

164 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

165 self.catalog.writeFits(filename) 

166 catalog = lsst.afw.table.SourceCatalog.readFits(filename) 

167 table = catalog.getTable() 

168 record = catalog[0] 

169 # I'm using the keys from the non-persisted table. They should work at least in the 

170 # current implementation 

171 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux()) 

172 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag()) 

173 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

174 centroid = self.centroidKey.get(self.record) 

175 self.assertEqual(centroid, record.getCentroid()) 

176 self.assertFloatsAlmostEqual( 

177 math.fabs(self.record.get(self.xErrKey)), 

178 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

179 self.assertFloatsAlmostEqual( 

180 math.fabs(self.record.get(self.yErrKey)), 

181 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

182 shape = self.shapeKey.get(self.record) 

183 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c") 

184 self.assertEqual(shape, record.getShape()) 

185 self.assertFloatsAlmostEqual( 

186 math.fabs(self.record.get(self.xxErrKey)), 

187 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

188 self.assertFloatsAlmostEqual( 

189 math.fabs(self.record.get(self.yyErrKey)), 

190 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

191 self.assertFloatsAlmostEqual( 

192 math.fabs(self.record.get(self.xyErrKey)), 

193 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

194 psfShape = self.psfShapeKey.get(self.record) 

195 self.assertEqual(table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

196 self.assertEqual(psfShape, record.getPsfShape()) 

197 

198 def testCanonical2(self): 

199 self.table.definePsfFlux("a") 

200 self.table.defineCentroid("b") 

201 self.table.defineShape("c") 

202 self.table.definePsfShape("d") 

203 self.checkCanonical() 

204 

205 def testPickle(self): 

206 p = pickle.dumps(self.catalog) 

207 new = pickle.loads(p) 

208 

209 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames()) 

210 self.assertEqual(len(self.catalog), len(new)) 

211 for r1, r2 in zip(self.catalog, new): 

212 # Columns that are easy to test 

213 for field in ("a_instFlux", "a_instFluxErr", "id"): 

214 k1 = self.catalog.schema.find(field).getKey() 

215 k2 = new.schema.find(field).getKey() 

216 self.assertEqual(r1[k1], r2[k2]) 

217 

218 def testCoordUpdate(self): 

219 self.table.defineCentroid("b") 

220 wcs = makeWcs() 

221 self.record.updateCoord(wcs) 

222 coord1 = self.record.getCoord() 

223 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey)) 

224 self.assertEqual(coord1, coord2) 

225 

226 def testSorting(self): 

227 self.assertFalse(self.catalog.isSorted()) 

228 self.catalog.sort() 

229 self.assertTrue(self.catalog.isSorted()) 

230 r = self.catalog.find(2) 

231 self.assertEqual(r["id"], 2) 

232 r = self.catalog.find(500) 

233 self.assertIsNone(r) 

234 

235 def testConversion(self): 

236 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog) 

237 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog) 

238 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True) 

239 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True) 

240 self.assertEqual(self.catalog.table, catalog1.table) 

241 self.assertEqual(self.catalog.table, catalog2.table) 

242 self.assertNotEqual(self.catalog.table, catalog3.table) 

243 self.assertNotEqual(self.catalog.table, catalog3.table) 

244 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4): 

245 self.assertEqual(r, r1) 

246 self.assertEqual(r, r2) 

247 self.assertNotEqual(r, r3) 

248 self.assertNotEqual(r, r4) 

249 self.assertEqual(r.getId(), r3.getId()) 

250 self.assertEqual(r.getId(), r4.getId()) 

251 

252 def testColumnView(self): 

253 cols1 = self.catalog.getColumnView() 

254 cols2 = self.catalog.columns 

255 self.assertIs(cols1, cols2) 

256 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView) 

257 self.table.definePsfFlux("a") 

258 self.table.defineCentroid("b") 

259 self.table.defineShape("c") 

260 self.table.definePsfShape("d") 

261 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux()) 

262 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr()) 

263 self.assertFloatsEqual(cols2["b_x"], cols2.getX()) 

264 self.assertFloatsEqual(cols2["b_y"], cols2.getY()) 

265 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx()) 

266 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy()) 

267 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy()) 

268 self.assertFloatsEqual(cols2["d_xx"], cols2.getPsfIxx()) 

269 self.assertFloatsEqual(cols2["d_yy"], cols2.getPsfIyy()) 

270 self.assertFloatsEqual(cols2["d_xy"], cols2.getPsfIxy()) 

271 

272 # Trying to access slots which have been removed should raise. 

273 self.catalog.table.schema.getAliasMap().erase("slot_Centroid") 

274 self.catalog.table.schema.getAliasMap().erase("slot_Shape") 

275 self.catalog.table.schema.getAliasMap().erase("slot_PsfShape") 

276 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy", "PsfIxx", "PsfIyy", "PsfIxy"]: 

277 with self.assertRaises(lsst.pex.exceptions.LogicError): 

278 getattr(self.catalog, f"get{quantity}")() 

279 

280 def testForwarding(self): 

281 """Verify that Catalog forwards unknown methods to its table and/or columns.""" 

282 self.table.definePsfFlux("a") 

283 self.table.defineCentroid("b") 

284 self.table.defineShape("c") 

285 self.assertFloatsEqual(self.catalog.columns["a_instFlux"], 

286 self.catalog["a_instFlux"]) 

287 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey], 

288 self.catalog.get(self.instFluxKey)) 

289 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey), 

290 self.catalog.getPsfInstFlux()) 

291 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey()) 

292 with self.assertRaises(AttributeError): 

293 self.catalog.foo() 

294 

295 def testBitsColumn(self): 

296 

297 allBits = self.catalog.getBits() 

298 someBits = self.catalog.getBits(["a_flag", "c_flag"]) 

299 self.assertEqual(allBits.getMask("a_flag"), 0x1) 

300 self.assertEqual(allBits.getMask("b_flag"), 0x2) 

301 self.assertEqual(allBits.getMask("c_flag"), 0x4) 

302 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1) 

303 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2) 

304 np.testing.assert_array_equal((allBits.array & 0x1 != 0), self.catalog["a_flag"]) 

305 np.testing.assert_array_equal((allBits.array & 0x2 != 0), self.catalog["b_flag"]) 

306 np.testing.assert_array_equal((allBits.array & 0x4 != 0), self.catalog["c_flag"]) 

307 np.testing.assert_array_equal((someBits.array & 0x1 != 0), self.catalog["a_flag"]) 

308 np.testing.assert_array_equal((someBits.array & 0x2 != 0), self.catalog["c_flag"]) 

309 

310 def testCast(self): 

311 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog) 

312 baseCat.cast(lsst.afw.table.SourceCatalog) 

313 

314 def testFootprints(self): 

315 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS 

316 ''' 

317 src1 = self.catalog.addNew() 

318 src2 = self.catalog.addNew() 

319 src3 = self.catalog.addNew() 

320 self.fillRecord(src1) 

321 self.fillRecord(src2) 

322 self.fillRecord(src3) 

323 src2.setParent(src1.getId()) 

324 

325 W, H = 100, 100 

326 mim = lsst.afw.image.MaskedImageF(W, H) 

327 im = mim.getImage() 

328 msk = mim.getMask() 

329 var = mim.getVariance() 

330 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int)) 

331 im.array[:] = y*1E6 + x*1E3 

332 msk.array[:] = (y << 8) | x 

333 var.array[:] = y*1E2 + x 

334 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50) 

335 circ = lsst.afw.detection.Footprint(spanSet) 

336 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim) 

337 src2.setFootprint(heavy) 

338 

339 for i, src in enumerate(self.catalog): 

340 if src != src2: 

341 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50) 

342 src.setFootprint(lsst.afw.detection.Footprint(spanSet)) 

343 

344 # insert this HeavyFootprint into an otherwise blank image (for comparing the results) 

345 mim2 = lsst.afw.image.MaskedImageF(W, H) 

346 heavy.insert(mim2) 

347 

348 with lsst.utils.tests.getTempFilePath(".fits") as fn: 

349 self.catalog.writeFits(fn) 

350 

351 cat2 = lsst.afw.table.SourceCatalog.readFits(fn) 

352 r2 = cat2[-2] 

353 h2 = r2.getFootprint() 

354 self.assertTrue(h2.isHeavy()) 

355 mim3 = lsst.afw.image.MaskedImageF(W, H) 

356 h2.insert(mim3) 

357 

358 self.assertFalse(cat2[-1].getFootprint().isHeavy()) 

359 self.assertFalse(cat2[-3].getFootprint().isHeavy()) 

360 self.assertFalse(cat2[0].getFootprint().isHeavy()) 

361 self.assertFalse(cat2[1].getFootprint().isHeavy()) 

362 self.assertFalse(cat2[2].getFootprint().isHeavy()) 

363 

364 if False: 

365 # Write out before-n-after FITS images 

366 for MI in [mim, mim2, mim3]: 

367 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits') 

368 os.close(f) 

369 MI.writeFits(fn2) 

370 print('wrote', fn2) 

371 

372 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray()) 

373 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray()) 

374 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray()) 

375 

376 im3 = mim3.getImage() 

377 ma3 = mim3.getMask() 

378 va3 = mim3.getVariance() 

379 for y in range(H): 

380 for x in range(W): 

381 if circ.contains(lsst.geom.Point2I(x, y)): 

382 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT]) 

383 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT]) 

384 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT]) 

385 else: 

386 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.) 

387 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.) 

388 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.) 

389 

390 cat3 = lsst.afw.table.SourceCatalog.readFits( 

391 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

392 for src in cat3: 

393 self.assertFalse(src.getFootprint().isHeavy()) 

394 cat4 = lsst.afw.table.SourceCatalog.readFits( 

395 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

396 for src in cat4: 

397 self.assertIsNone(src.getFootprint()) 

398 

399 self.catalog.writeFits( 

400 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

401 cat5 = lsst.afw.table.SourceCatalog.readFits(fn) 

402 for src in cat5: 

403 self.assertFalse(src.getFootprint().isHeavy()) 

404 

405 self.catalog.writeFits( 

406 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

407 cat6 = lsst.afw.table.SourceCatalog.readFits(fn) 

408 for src in cat6: 

409 self.assertIsNone(src.getFootprint()) 

410 

411 def testIdFactory(self): 

412 expId = int(1257198) 

413 reserved = 32 

414 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved) 

415 id1 = factory() 

416 id2 = factory() 

417 self.assertEqual(id2 - id1, 1) 

418 factory.notify(0xFFFFFFFF) 

419 with self.assertRaises(lsst.pex.exceptions.LengthError): 

420 factory() 

421 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

422 factory.notify(0x1FFFFFFFF) 

423 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

424 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved) 

425 

426 def testFamilies(self): 

427 self.catalog.sort() 

428 parents = self.catalog.getChildren(0) 

429 self.assertEqual(list(parents), list(self.catalog)) 

430 parentKey = lsst.afw.table.SourceTable.getParentKey() 

431 for parent in parents: 

432 self.assertEqual(parent.get(parentKey), 0) 

433 for i in range(10): 

434 child = self.catalog.addNew() 

435 self.fillRecord(child) 

436 child.set(parentKey, parent.getId()) 

437 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents], 

438 [record.getId() for record in self.catalog]) 

439 for parent, (children, ids) in zip(parents, childrenIter): 

440 self.assertEqual(len(children), 10) 

441 self.assertEqual(len(children), len(ids)) 

442 for child, id in zip(children, ids): 

443 self.assertEqual(child.getParent(), parent.getId()) 

444 self.assertEqual(child.getId(), id) 

445 

446 # Check detection of unsorted catalog 

447 self.catalog.sort(self.instFluxKey) 

448 with self.assertRaises(AssertionError): 

449 self.catalog.getChildren(0) 

450 self.catalog.sort(parentKey) 

451 self.catalog.getChildren(0) # Just care this succeeds 

452 

453 def testFitsReadVersion0Compatibility(self): 

454 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits")) 

455 self.assertTrue(cat.getPsfFluxSlot().isValid()) 

456 self.assertTrue(cat.getApFluxSlot().isValid()) 

457 self.assertTrue(cat.getGaussianFluxSlot().isValid()) 

458 self.assertTrue(cat.getModelFluxSlot().isValid()) 

459 self.assertTrue(cat.getCentroidSlot().isValid()) 

460 self.assertTrue(cat.getShapeSlot().isValid()) 

461 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(), 

462 cat.schema.find("flux_psf").key) 

463 self.assertEqual(cat.getApFluxSlot().getMeasKey(), 

464 cat.schema.find("flux_sinc").key) 

465 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(), 

466 cat.schema.find("flux_naive").key) 

467 self.assertEqual(cat.getModelFluxSlot().getMeasKey(), 

468 cat.schema.find("cmodel_flux").key) 

469 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(), 

470 cat.schema.find("centroid_sdss_x").key) 

471 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(), 

472 cat.schema.find("centroid_sdss_y").key) 

473 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(), 

474 cat.schema.find("shape_hsm_moments_xx").key) 

475 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(), 

476 cat.schema.find("shape_hsm_moments_yy").key) 

477 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(), 

478 cat.schema.find("shape_hsm_moments_xy").key) 

479 self.assertEqual(cat.getPsfFluxSlot().getErrKey(), 

480 cat.schema.find("flux_psf_err").key) 

481 self.assertEqual(cat.getApFluxSlot().getErrKey(), 

482 cat.schema.find("flux_sinc_err").key) 

483 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(), 

484 cat.schema.find("flux_naive_err").key) 

485 self.assertEqual(cat.getModelFluxSlot().getErrKey(), 

486 cat.schema.find("cmodel_flux_err").key) 

487 self.assertEqual( 

488 cat.getCentroidSlot().getErrKey(), 

489 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"])) 

490 self.assertEqual( 

491 cat.getShapeSlot().getErrKey(), 

492 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"])) 

493 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(), 

494 cat.schema.find("flux_psf_flags").key) 

495 self.assertEqual(cat.getApFluxSlot().getFlagKey(), 

496 cat.schema.find("flux_sinc_flags").key) 

497 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(), 

498 cat.schema.find("flux_naive_flags").key) 

499 self.assertEqual(cat.getModelFluxSlot().getFlagKey(), 

500 cat.schema.find("cmodel_flux_flags").key) 

501 self.assertEqual(cat.getCentroidSlot().getFlagKey(), 

502 cat.schema.find("centroid_sdss_flags").key) 

503 self.assertEqual(cat.getShapeSlot().getFlagKey(), 

504 cat.schema.find("shape_hsm_moments_flags").key) 

505 

506 def testFitsReadVersion1Compatibility(self): 

507 """Test reading of catalogs with version 1 schema 

508 

509 Version 1 catalogs need to have added aliases from Sigma->Err and 

510 from `_flux`->`_instFlux`. 

511 """ 

512 cat = lsst.afw.table.SourceCatalog.readFits( 

513 os.path.join(testPath, "data", "sourceTable-v1.fits")) 

514 self.assertEqual( 

515 cat.getCentroidSlot().getErrKey(), 

516 lsst.afw.table.CovarianceMatrix2fKey( 

517 cat.schema["slot_Centroid"], 

518 ["x", "y"])) 

519 self.assertEqual( 

520 cat.getShapeSlot().getErrKey(), 

521 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"])) 

522 # check the flux->instFlux conversion 

523 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

524 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

525 

526 def testFitsReadVersion2CompatibilityRealSourceCatalog(self): 

527 """DM-15891: some fields were getting aliases they shouldn't have.""" 

528 cat = lsst.afw.table.SourceCatalog.readFits( 

529 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits")) 

530 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema) 

531 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema) 

532 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema) 

533 

534 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self): 

535 """DM-16068: some fields were not getting aliases they should have 

536 

537 In particular, the alias setting relies on flux fields having their 

538 units set properly. Prior to a resolution of DM-16068, the units 

539 were not getting set for several CModel flux fields and one deblender 

540 field (deblend_psfFlux), and thus were not getting the 

541 `_flux`->`_instFlux` aliases set. 

542 

543 NOTE: this test will (is meant to) fail on the read until DM-16068 is 

544 resolved. The error is: 

545 

546 lsst::pex::exceptions::NotFoundError: 'Field or subfield with 

547 name 'modelfit_CModel_instFlux' not found with type 'D'.' 

548 """ 

549 cat = lsst.afw.table.SourceCatalog.readFits( 

550 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits")) 

551 self.assertIn('modelfit_CModel_instFlux', cat.schema) 

552 self.assertIn('modelfit_CModel_instFluxErr', cat.schema) 

553 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema) 

554 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema) 

555 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema) 

556 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema) 

557 

558 def testFitsVersion2Compatibility(self): 

559 """Test reading of catalogs with version 2 schema 

560 

561 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`. 

562 """ 

563 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits")) 

564 # check the flux->instFlux conversion 

565 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

566 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

567 

568 def testDM1083(self): 

569 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

570 st = lsst.afw.table.SourceTable.make(schema) 

571 cat = lsst.afw.table.SourceCatalog(st) 

572 tmp = lsst.afw.table.SourceCatalog(cat.getTable()) 

573 record = tmp.addNew() 

574 cat.extend(tmp) 

575 self.assertEqual(cat[0].getId(), record.getId()) 

576 # check that the same record is in both catalogs (not a copy) 

577 record.setId(15) 

578 self.assertEqual(cat[0].getId(), record.getId()) 

579 

580 def testSlotUndefine(self): 

581 """Test that we can correctly define and undefine a slot after a SourceTable has been created""" 

582 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

583 key = schema.addField("a_instFlux", type=np.float64, doc="flux field") 

584 table = lsst.afw.table.SourceTable.make(schema) 

585 table.definePsfFlux("a") 

586 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key) 

587 table.schema.getAliasMap().erase("slot_PsfFlux") 

588 self.assertFalse(table.getPsfFluxSlot().isValid()) 

589 

590 def testOldFootprintPersistence(self): 

591 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older 

592 version of the pipeline with a different format. 

593 """ 

594 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits") 

595 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename) 

596 self.assertEqual(len(catalog1), 2) 

597 with self.assertRaises(LookupError): 

598 catalog1.schema.find("footprint") 

599 fp1 = catalog1[0].getFootprint() 

600 fp2 = catalog1[1].getFootprint() 

601 self.assertEqual(fp1.getArea(), 495) 

602 self.assertEqual(fp2.getArea(), 767) 

603 self.assertFalse(fp1.isHeavy()) 

604 self.assertTrue(fp2.isHeavy()) 

605 self.assertEqual(len(fp1.getSpans()), 29) 

606 self.assertEqual(len(fp2.getSpans()), 44) 

607 self.assertEqual(len(fp1.getPeaks()), 1) 

608 self.assertEqual(len(fp2.getPeaks()), 1) 

609 self.assertEqual(fp1.getBBox(), 

610 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29))) 

611 self.assertEqual(fp2.getBBox(), 

612 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38))) 

613 hfp = lsst.afw.detection.HeavyFootprintF(fp2) 

614 self.assertEqual(len(hfp.getImageArray()), fp2.getArea()) 

615 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea()) 

616 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea()) 

617 catalog2 = lsst.afw.table.SourceCatalog.readFits( 

618 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

619 self.assertEqual(list(fp1.getSpans()), 

620 list(catalog2[0].getFootprint().getSpans())) 

621 self.assertEqual(list(fp2.getSpans()), 

622 list(catalog2[1].getFootprint().getSpans())) 

623 self.assertFalse(catalog2[1].getFootprint().isHeavy()) 

624 catalog3 = lsst.afw.table.SourceCatalog.readFits( 

625 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

626 self.assertEqual(catalog3[0].getFootprint(), None) 

627 self.assertEqual(catalog3[1].getFootprint(), None) 

628 

629 def _testFluxSlot(self, slotName): 

630 """Demonstrate that we can create & use the named Flux slot.""" 

631 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

632 baseName = "afw_Test" 

633 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux") 

634 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty") 

635 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag") 

636 catalog = lsst.afw.table.SourceCatalog(schema) 

637 table = catalog.table 

638 

639 # Initially, the slot is undefined. 

640 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

641 

642 # After definition, it maps to the keys defined above. 

643 getattr(table, f"define{slotName}")(baseName) 

644 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid()) 

645 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

646 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

647 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

648 

649 # We should be able to retrieve arbitrary values set in records. 

650 record = catalog.addNew() 

651 instFlux, err, flag = 10.0, 1.0, False 

652 record.set(instFluxKey, instFlux) 

653 record.set(errKey, err) 

654 record.set(flagKey, flag) 

655 instFluxName = slotName.replace("Flux", "InstFlux") 

656 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux) 

657 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err) 

658 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag) 

659 

660 # Should also be able to retrieve them as columns from the catalog 

661 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux) 

662 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err) 

663 

664 # And we should be able to delete the slot, breaking the mapping. 

665 table.schema.getAliasMap().erase(f"slot_{slotName}") 

666 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

667 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

668 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

669 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

670 

671 # When the slot has been deleted, attempting to access it should 

672 # throw a LogicError. 

673 with self.assertRaises(lsst.pex.exceptions.LogicError): 

674 getattr(catalog, f"get{instFluxName}")() 

675 with self.assertRaises(lsst.pex.exceptions.LogicError): 

676 getattr(catalog, f"get{instFluxName}Err")() 

677 

678 def testFluxSlots(self): 

679 """Check that all the expected flux slots are present & correct.""" 

680 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux", 

681 "PsfFlux"]: 

682 self._testFluxSlot(slotName) 

683 

684 # But, of course, we should not accept a slot which hasn't be defined. 

685 with self.assertRaises(AttributeError): 

686 self._testFluxSlot("NotExtantFlux") 

687 

688 def testStr(self): 

689 """Check that the str() produced on a catalog contains expected things.""" 

690 string = str(self.catalog) 

691 for field in ('id', 'coord_ra', 'coord_dec'): 

692 self.assertIn(field, string) 

693 

694 def testRepr(self): 

695 """Check that the repr() produced on a catalog contains expected things.""" 

696 string = repr(self.catalog) 

697 self.assertIn(str(type(self.catalog)), string) 

698 for field in ('id', 'coord_ra', 'coord_dec'): 

699 self.assertIn(field, string) 

700 

701 def testStrNonContiguous(self): 

702 """Check that str() doesn't fail on non-contiguous tables.""" 

703 del self.catalog[1] 

704 string = str(self.catalog) 

705 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string) 

706 for field in ('id', 'coord_ra', 'coord_dec'): 

707 self.assertIn(field, string) 

708 

709 def testRecordStr(self): 

710 """Test that str(record) contains expected things.""" 

711 string = str(self.catalog[0]) 

712 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

713 self.assertIn(field, string) 

714 

715 def testRecordRepr(self): 

716 """Test that repr(record) contains expected things.""" 

717 string = repr(self.catalog[0]) 

718 self.assertIn(str(type(self.catalog[0])), string) 

719 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

720 self.assertIn(field, string) 

721 

722 def testGetNonContiguous(self): 

723 """Check that we can index on non-contiguous tables""" 

724 # Make a non-contiguous catalog 

725 nonContiguous = type(self.catalog)(self.catalog.table) 

726 for rr in reversed(self.catalog): 

727 nonContiguous.append(rr) 

728 num = len(self.catalog) 

729 # Check assumptions 

730 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog 

731 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique 

732 # Indexing with boolean array 

733 select = np.zeros(num, dtype=bool) 

734 select[1] = True 

735 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"]) 

736 # Extracting a number column 

737 column = "a_instFlux" 

738 array = nonContiguous[column] 

739 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column]) 

740 with self.assertRaises(ValueError): 

741 array[1] = 1.2345 # Should be immutable 

742 # Extracting a flag column 

743 column = "a_flag" 

744 array = nonContiguous[column] 

745 np.testing.assert_equal(np.flip(array, 0), self.catalog[column]) 

746 with self.assertRaises(ValueError): 

747 array[1] = True # Should be immutable 

748 

749 

750class MemoryTester(lsst.utils.tests.MemoryTestCase): 

751 pass 

752 

753 

754def setup_module(module): 

755 lsst.utils.tests.init() 

756 

757 

758if __name__ == "__main__": 758 ↛ 759line 758 didn't jump to line 759, because the condition on line 758 was never true

759 lsst.utils.tests.init() 

760 unittest.main()