Coverage for tests/test_sourceTable.py: 9%

527 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-04-15 02:24 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import tempfile 

25import pickle 

26import math 

27 

28import numpy as np 

29 

30import lsst.utils.tests 

31import lsst.pex.exceptions 

32import lsst.geom 

33import lsst.afw.table 

34import lsst.afw.geom 

35import lsst.afw.image 

36import lsst.afw.detection 

37 

38testPath = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41def makeArray(size, dtype): 

42 return np.array(np.random.randn(*size), dtype=dtype) 

43 

44 

45def makeCov(size, dtype): 

46 m = np.array(np.random.randn(size, size), dtype=dtype) 

47 return np.dot(m, m.transpose()) 

48 

49 

50def makeWcs(): 

51 crval = lsst.geom.SpherePoint(1.606631*lsst.geom.degrees, 

52 5.090329*lsst.geom.degrees) 

53 crpix = lsst.geom.Point2D(2036.0, 2000.0) 

54 cdMatrix = np.array([5.399452e-5, -1.30770e-5, 1.30770e-5, 5.399452e-5]) 

55 cdMatrix.shape = (2, 2) 

56 return lsst.afw.geom.makeSkyWcs(crpix=crpix, crval=crval, cdMatrix=cdMatrix) 

57 

58 

59class SourceTableTestCase(lsst.utils.tests.TestCase): 

60 

61 def fillRecord(self, record): 

62 record.set(self.instFluxKey, np.random.randn()) 

63 record.set(self.instFluxErrKey, np.random.randn()) 

64 record.set(self.centroidKey.getX(), np.random.randn()) 

65 record.set(self.centroidKey.getY(), np.random.randn()) 

66 record.set(self.xErrKey, np.random.randn()) 

67 record.set(self.yErrKey, np.random.randn()) 

68 record.set(self.shapeKey.getIxx(), np.random.randn()) 

69 record.set(self.shapeKey.getIyy(), np.random.randn()) 

70 record.set(self.shapeKey.getIxy(), np.random.randn()) 

71 record.set(self.xxErrKey, np.random.randn()) 

72 record.set(self.yyErrKey, np.random.randn()) 

73 record.set(self.xyErrKey, np.random.randn()) 

74 record.set(self.psfShapeKey.getIxx(), np.random.randn()) 

75 record.set(self.psfShapeKey.getIyy(), np.random.randn()) 

76 record.set(self.psfShapeKey.getIxy(), np.random.randn()) 

77 record.set(self.fluxFlagKey, np.random.randn() > 0) 

78 record.set(self.centroidFlagKey, np.random.randn() > 0) 

79 record.set(self.shapeFlagKey, np.random.randn() > 0) 

80 

81 def setUp(self): 

82 np.random.seed(1) 

83 self.schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

84 self.instFluxKey = self.schema.addField("a_instFlux", type="D") 

85 self.instFluxErrKey = self.schema.addField("a_instFluxErr", type="D") 

86 self.fluxFlagKey = self.schema.addField("a_flag", type="Flag") 

87 

88 # the meas field is added using a functor key, but the error is added 

89 # as scalars, as we lack a ResultKey functor as exists in meas_base 

90 self.centroidKey = lsst.afw.table.Point2DKey.addFields( 

91 self.schema, "b", "", "pixel") 

92 self.xErrKey = self.schema.addField("b_xErr", type="F") 

93 self.yErrKey = self.schema.addField("b_yErr", type="F") 

94 self.centroidFlagKey = self.schema.addField("b_flag", type="Flag") 

95 

96 self.shapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

97 self.schema, "c", "", lsst.afw.table.CoordinateType.PIXEL) 

98 self.xxErrKey = self.schema.addField("c_xxErr", type="F") 

99 self.xyErrKey = self.schema.addField("c_xyErr", type="F") 

100 self.yyErrKey = self.schema.addField("c_yyErr", type="F") 

101 self.shapeFlagKey = self.schema.addField("c_flag", type="Flag") 

102 

103 self.psfShapeKey = lsst.afw.table.QuadrupoleKey.addFields( 

104 self.schema, "d", "", lsst.afw.table.CoordinateType.PIXEL) 

105 self.psfShapeFlagKey = self.schema.addField("d_flag", type="Flag") 

106 

107 self.coordErrKey = lsst.afw.table.CoordKey.addErrorFields(self.schema) 

108 

109 self.table = lsst.afw.table.SourceTable.make(self.schema) 

110 self.catalog = lsst.afw.table.SourceCatalog(self.table) 

111 self.record = self.catalog.addNew() 

112 self.fillRecord(self.record) 

113 self.record.setId(50) 

114 self.fillRecord(self.catalog.addNew()) 

115 self.fillRecord(self.catalog.addNew()) 

116 

117 def tearDown(self): 

118 del self.schema 

119 del self.record 

120 del self.table 

121 del self.catalog 

122 

123 def checkCanonical(self): 

124 self.assertEqual(self.record.get(self.instFluxKey), 

125 self.record.getPsfInstFlux()) 

126 self.assertEqual(self.record.get(self.fluxFlagKey), 

127 self.record.getPsfFluxFlag()) 

128 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

129 self.assertEqual(self.centroidKey.get(self.record), 

130 self.record.getCentroid()) 

131 self.assertFloatsAlmostEqual( 

132 math.fabs(self.record.get(self.xErrKey)), 

133 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

134 self.assertFloatsAlmostEqual( 

135 math.fabs(self.record.get(self.yErrKey)), 

136 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

137 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_Shape"), "c") 

138 self.assertEqual(self.shapeKey.get(self.record), 

139 self.record.getShape()) 

140 self.assertFloatsAlmostEqual( 

141 math.fabs(self.record.get(self.xxErrKey)), 

142 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

143 self.assertFloatsAlmostEqual( 

144 math.fabs(self.record.get(self.yyErrKey)), 

145 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

146 self.assertFloatsAlmostEqual( 

147 math.fabs(self.record.get(self.xyErrKey)), 

148 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

149 self.assertEqual(self.table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

150 self.assertEqual(self.psfShapeKey.get(self.record), 

151 self.record.getPsfShape()) 

152 

153 def testPersisted(self): 

154 self.table.definePsfFlux("a") 

155 self.table.defineCentroid("b") 

156 self.table.defineShape("c") 

157 self.table.definePsfShape("d") 

158 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

159 self.catalog.writeFits(filename) 

160 catalog = lsst.afw.table.SourceCatalog.readFits(filename) 

161 table = catalog.getTable() 

162 record = catalog[0] 

163 # I'm using the keys from the non-persisted table. They should work at least in the 

164 # current implementation 

165 self.assertEqual(record.get(self.instFluxKey), record.getPsfInstFlux()) 

166 self.assertEqual(record.get(self.fluxFlagKey), record.getPsfFluxFlag()) 

167 self.assertEqual(table.getSchema().getAliasMap().get("slot_Centroid"), "b") 

168 centroid = self.centroidKey.get(self.record) 

169 self.assertEqual(centroid, record.getCentroid()) 

170 self.assertFloatsAlmostEqual( 

171 math.fabs(self.record.get(self.xErrKey)), 

172 math.sqrt(self.record.getCentroidErr()[0, 0]), rtol=1e-6) 

173 self.assertFloatsAlmostEqual( 

174 math.fabs(self.record.get(self.yErrKey)), 

175 math.sqrt(self.record.getCentroidErr()[1, 1]), rtol=1e-6) 

176 shape = self.shapeKey.get(self.record) 

177 self.assertEqual(table.getSchema().getAliasMap().get("slot_Shape"), "c") 

178 self.assertEqual(shape, record.getShape()) 

179 self.assertFloatsAlmostEqual( 

180 math.fabs(self.record.get(self.xxErrKey)), 

181 math.sqrt(self.record.getShapeErr()[0, 0]), rtol=1e-6) 

182 self.assertFloatsAlmostEqual( 

183 math.fabs(self.record.get(self.yyErrKey)), 

184 math.sqrt(self.record.getShapeErr()[1, 1]), rtol=1e-6) 

185 self.assertFloatsAlmostEqual( 

186 math.fabs(self.record.get(self.xyErrKey)), 

187 math.sqrt(self.record.getShapeErr()[2, 2]), rtol=1e-6) 

188 psfShape = self.psfShapeKey.get(self.record) 

189 self.assertEqual(table.getSchema().getAliasMap().get("slot_PsfShape"), "d") 

190 self.assertEqual(psfShape, record.getPsfShape()) 

191 

192 def testCanonical2(self): 

193 self.table.definePsfFlux("a") 

194 self.table.defineCentroid("b") 

195 self.table.defineShape("c") 

196 self.table.definePsfShape("d") 

197 self.checkCanonical() 

198 

199 def testPickle(self): 

200 p = pickle.dumps(self.catalog) 

201 new = pickle.loads(p) 

202 

203 self.assertEqual(self.catalog.schema.getNames(), new.schema.getNames()) 

204 self.assertEqual(len(self.catalog), len(new)) 

205 for r1, r2 in zip(self.catalog, new): 

206 # Columns that are easy to test 

207 for field in ("a_instFlux", "a_instFluxErr", "id"): 

208 k1 = self.catalog.schema.find(field).getKey() 

209 k2 = new.schema.find(field).getKey() 

210 self.assertEqual(r1[k1], r2[k2]) 

211 

212 def testCoordUpdate(self): 

213 self.table.defineCentroid("b") 

214 wcs = makeWcs() 

215 self.record.updateCoord(wcs) 

216 coord1 = self.record.getCoord() 

217 coord2 = wcs.pixelToSky(self.record.get(self.centroidKey)) 

218 self.assertEqual(coord1, coord2) 

219 

220 def testCoordErrors(self): 

221 self.table.defineCentroid("b") 

222 wcs = makeWcs() 

223 self.record.updateCoord(wcs) 

224 

225 scale = (1.0 * lsst.geom.arcseconds).asDegrees() 

226 center = self.record.getCentroid() 

227 skyCenter = wcs.pixelToSky(center) 

228 localGnomonicWcs = lsst.afw.geom.makeSkyWcs( 

229 center, skyCenter, np.diag((scale, scale))) 

230 measurementToLocalGnomonic = wcs.getTransform().then( 

231 localGnomonicWcs.getTransform().inverted() 

232 ) 

233 localMatrix = measurementToLocalGnomonic.getJacobian(center) 

234 radMatrix = np.radians(localMatrix / 3600) 

235 

236 centroidErr = self.record.getCentroidErr() 

237 coordErr = radMatrix.dot(centroidErr.dot(radMatrix.T)) 

238 # multiply RA by cos(Dec): 

239 cosDec = np.cos(skyCenter.getDec().asRadians()) 

240 decCorr = np.array([[cosDec, 0], [0, 1]]) 

241 coordErrCorr = decCorr.dot(coordErr.dot(decCorr)) 

242 catCoordErr = self.record.get(self.coordErrKey) 

243 np.testing.assert_almost_equal(coordErrCorr, catCoordErr, decimal=16) 

244 

245 def testSorting(self): 

246 self.assertFalse(self.catalog.isSorted()) 

247 self.catalog.sort() 

248 self.assertTrue(self.catalog.isSorted()) 

249 r = self.catalog.find(2) 

250 self.assertEqual(r["id"], 2) 

251 r = self.catalog.find(500) 

252 self.assertIsNone(r) 

253 

254 def testConversion(self): 

255 catalog1 = self.catalog.cast(lsst.afw.table.SourceCatalog) 

256 catalog2 = self.catalog.cast(lsst.afw.table.SimpleCatalog) 

257 catalog3 = self.catalog.cast(lsst.afw.table.SourceCatalog, deep=True) 

258 catalog4 = self.catalog.cast(lsst.afw.table.SimpleCatalog, deep=True) 

259 self.assertEqual(self.catalog.table, catalog1.table) 

260 self.assertEqual(self.catalog.table, catalog2.table) 

261 self.assertNotEqual(self.catalog.table, catalog3.table) 

262 self.assertNotEqual(self.catalog.table, catalog3.table) 

263 for r, r1, r2, r3, r4 in zip(self.catalog, catalog1, catalog2, catalog3, catalog4): 

264 self.assertEqual(r, r1) 

265 self.assertEqual(r, r2) 

266 self.assertNotEqual(r, r3) 

267 self.assertNotEqual(r, r4) 

268 self.assertEqual(r.getId(), r3.getId()) 

269 self.assertEqual(r.getId(), r4.getId()) 

270 

271 def testColumnView(self): 

272 cols1 = self.catalog.getColumnView() 

273 cols2 = self.catalog.columns 

274 self.assertIs(cols1, cols2) 

275 self.assertIsInstance(cols1, lsst.afw.table.SourceColumnView) 

276 self.table.definePsfFlux("a") 

277 self.table.defineCentroid("b") 

278 self.table.defineShape("c") 

279 self.table.definePsfShape("d") 

280 self.assertFloatsEqual(cols2["a_instFlux"], cols2.getPsfInstFlux()) 

281 self.assertFloatsEqual(cols2["a_instFluxErr"], cols2.getPsfInstFluxErr()) 

282 self.assertFloatsEqual(cols2["b_x"], cols2.getX()) 

283 self.assertFloatsEqual(cols2["b_y"], cols2.getY()) 

284 self.assertFloatsEqual(cols2["c_xx"], cols2.getIxx()) 

285 self.assertFloatsEqual(cols2["c_yy"], cols2.getIyy()) 

286 self.assertFloatsEqual(cols2["c_xy"], cols2.getIxy()) 

287 self.assertFloatsEqual(cols2["d_xx"], cols2.getPsfIxx()) 

288 self.assertFloatsEqual(cols2["d_yy"], cols2.getPsfIyy()) 

289 self.assertFloatsEqual(cols2["d_xy"], cols2.getPsfIxy()) 

290 

291 # Trying to access slots which have been removed should raise. 

292 self.catalog.table.schema.getAliasMap().erase("slot_Centroid") 

293 self.catalog.table.schema.getAliasMap().erase("slot_Shape") 

294 self.catalog.table.schema.getAliasMap().erase("slot_PsfShape") 

295 for quantity in ["X", "Y", "Ixx", "Iyy", "Ixy", "PsfIxx", "PsfIyy", "PsfIxy"]: 

296 with self.assertRaises(lsst.pex.exceptions.LogicError): 

297 getattr(self.catalog, f"get{quantity}")() 

298 

299 def testForwarding(self): 

300 """Verify that Catalog forwards unknown methods to its table and/or columns.""" 

301 self.table.definePsfFlux("a") 

302 self.table.defineCentroid("b") 

303 self.table.defineShape("c") 

304 self.assertFloatsEqual(self.catalog.columns["a_instFlux"], 

305 self.catalog["a_instFlux"]) 

306 self.assertFloatsEqual(self.catalog.columns[self.instFluxKey], 

307 self.catalog.get(self.instFluxKey)) 

308 self.assertFloatsEqual(self.catalog.columns.get(self.instFluxKey), 

309 self.catalog.getPsfInstFlux()) 

310 self.assertEqual(self.instFluxKey, self.catalog.getPsfFluxSlot().getMeasKey()) 

311 with self.assertRaises(AttributeError): 

312 self.catalog.foo() 

313 

314 def testBitsColumn(self): 

315 

316 allBits = self.catalog.getBits() 

317 someBits = self.catalog.getBits(["a_flag", "c_flag"]) 

318 self.assertEqual(allBits.getMask("a_flag"), 0x1) 

319 self.assertEqual(allBits.getMask("b_flag"), 0x2) 

320 self.assertEqual(allBits.getMask("c_flag"), 0x4) 

321 self.assertEqual(someBits.getMask(self.fluxFlagKey), 0x1) 

322 self.assertEqual(someBits.getMask(self.shapeFlagKey), 0x2) 

323 np.testing.assert_array_equal((allBits.array & 0x1 != 0), self.catalog["a_flag"]) 

324 np.testing.assert_array_equal((allBits.array & 0x2 != 0), self.catalog["b_flag"]) 

325 np.testing.assert_array_equal((allBits.array & 0x4 != 0), self.catalog["c_flag"]) 

326 np.testing.assert_array_equal((someBits.array & 0x1 != 0), self.catalog["a_flag"]) 

327 np.testing.assert_array_equal((someBits.array & 0x2 != 0), self.catalog["c_flag"]) 

328 

329 def testCast(self): 

330 baseCat = self.catalog.cast(lsst.afw.table.BaseCatalog) 

331 baseCat.cast(lsst.afw.table.SourceCatalog) 

332 

333 def testFootprints(self): 

334 '''Test round-tripping Footprints (inc. HeavyFootprints) to FITS 

335 ''' 

336 src1 = self.catalog.addNew() 

337 src2 = self.catalog.addNew() 

338 src3 = self.catalog.addNew() 

339 self.fillRecord(src1) 

340 self.fillRecord(src2) 

341 self.fillRecord(src3) 

342 src2.setParent(src1.getId()) 

343 

344 W, H = 100, 100 

345 mim = lsst.afw.image.MaskedImageF(W, H) 

346 im = mim.getImage() 

347 msk = mim.getMask() 

348 var = mim.getVariance() 

349 x, y = np.meshgrid(np.arange(W, dtype=int), np.arange(H, dtype=int)) 

350 im.array[:] = y*1E6 + x*1E3 

351 msk.array[:] = (y << 8) | x 

352 var.array[:] = y*1E2 + x 

353 spanSet = lsst.afw.geom.SpanSet.fromShape(20).shiftedBy(50, 50) 

354 circ = lsst.afw.detection.Footprint(spanSet) 

355 heavy = lsst.afw.detection.makeHeavyFootprint(circ, mim) 

356 src2.setFootprint(heavy) 

357 

358 for i, src in enumerate(self.catalog): 

359 if src != src2: 

360 spanSet = lsst.afw.geom.SpanSet.fromShape(1 + i*2).shiftedBy(50, 50) 

361 src.setFootprint(lsst.afw.detection.Footprint(spanSet)) 

362 

363 # insert this HeavyFootprint into an otherwise blank image (for comparing the results) 

364 mim2 = lsst.afw.image.MaskedImageF(W, H) 

365 heavy.insert(mim2) 

366 

367 with lsst.utils.tests.getTempFilePath(".fits") as fn: 

368 self.catalog.writeFits(fn) 

369 

370 cat2 = lsst.afw.table.SourceCatalog.readFits(fn) 

371 r2 = cat2[-2] 

372 h2 = r2.getFootprint() 

373 self.assertTrue(h2.isHeavy()) 

374 mim3 = lsst.afw.image.MaskedImageF(W, H) 

375 h2.insert(mim3) 

376 

377 self.assertFalse(cat2[-1].getFootprint().isHeavy()) 

378 self.assertFalse(cat2[-3].getFootprint().isHeavy()) 

379 self.assertFalse(cat2[0].getFootprint().isHeavy()) 

380 self.assertFalse(cat2[1].getFootprint().isHeavy()) 

381 self.assertFalse(cat2[2].getFootprint().isHeavy()) 

382 

383 if False: 

384 # Write out before-n-after FITS images 

385 for MI in [mim, mim2, mim3]: 

386 f, fn2 = tempfile.mkstemp(prefix='testHeavyFootprint-', suffix='.fits') 

387 os.close(f) 

388 MI.writeFits(fn2) 

389 print('wrote', fn2) 

390 

391 self.assertFloatsEqual(mim2.getImage().getArray(), mim3.getImage().getArray()) 

392 self.assertFloatsEqual(mim2.getMask().getArray(), mim3.getMask().getArray()) 

393 self.assertFloatsEqual(mim2.getVariance().getArray(), mim3.getVariance().getArray()) 

394 

395 im3 = mim3.getImage() 

396 ma3 = mim3.getMask() 

397 va3 = mim3.getVariance() 

398 for y in range(H): 

399 for x in range(W): 

400 if circ.contains(lsst.geom.Point2I(x, y)): 

401 self.assertEqual(im[x, y, lsst.afw.image.PARENT], im3[x, y, lsst.afw.image.PARENT]) 

402 self.assertEqual(msk[x, y, lsst.afw.image.PARENT], ma3[x, y, lsst.afw.image.PARENT]) 

403 self.assertEqual(var[x, y, lsst.afw.image.PARENT], va3[x, y, lsst.afw.image.PARENT]) 

404 else: 

405 self.assertEqual(im3[x, y, lsst.afw.image.PARENT], 0.) 

406 self.assertEqual(ma3[x, y, lsst.afw.image.PARENT], 0.) 

407 self.assertEqual(va3[x, y, lsst.afw.image.PARENT], 0.) 

408 

409 cat3 = lsst.afw.table.SourceCatalog.readFits( 

410 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

411 for src in cat3: 

412 self.assertFalse(src.getFootprint().isHeavy()) 

413 cat4 = lsst.afw.table.SourceCatalog.readFits( 

414 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

415 for src in cat4: 

416 self.assertIsNone(src.getFootprint()) 

417 

418 self.catalog.writeFits( 

419 fn, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

420 cat5 = lsst.afw.table.SourceCatalog.readFits(fn) 

421 for src in cat5: 

422 self.assertFalse(src.getFootprint().isHeavy()) 

423 

424 self.catalog.writeFits( 

425 fn, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

426 cat6 = lsst.afw.table.SourceCatalog.readFits(fn) 

427 for src in cat6: 

428 self.assertIsNone(src.getFootprint()) 

429 

430 def testIdFactory(self): 

431 expId = int(1257198) 

432 reserved = 32 

433 factory = lsst.afw.table.IdFactory.makeSource(expId, reserved) 

434 id1 = factory() 

435 id2 = factory() 

436 self.assertEqual(id2 - id1, 1) 

437 factory.notify(0xFFFFFFFF) 

438 with self.assertRaises(lsst.pex.exceptions.LengthError): 

439 factory() 

440 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

441 factory.notify(0x1FFFFFFFF) 

442 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

443 lsst.afw.table.IdFactory.makeSource(0x1FFFFFFFF, reserved) 

444 

445 def testFamilies(self): 

446 self.catalog.sort() 

447 parents = self.catalog.getChildren(0) 

448 self.assertEqual(list(parents), list(self.catalog)) 

449 parentKey = lsst.afw.table.SourceTable.getParentKey() 

450 for parent in parents: 

451 self.assertEqual(parent.get(parentKey), 0) 

452 for i in range(10): 

453 child = self.catalog.addNew() 

454 self.fillRecord(child) 

455 child.set(parentKey, parent.getId()) 

456 childrenIter = self.catalog.getChildren([parent.getId() for parent in parents], 

457 [record.getId() for record in self.catalog]) 

458 for parent, (children, ids) in zip(parents, childrenIter): 

459 self.assertEqual(len(children), 10) 

460 self.assertEqual(len(children), len(ids)) 

461 for child, id in zip(children, ids): 

462 self.assertEqual(child.getParent(), parent.getId()) 

463 self.assertEqual(child.getId(), id) 

464 

465 # Check detection of unsorted catalog 

466 self.catalog.sort(self.instFluxKey) 

467 with self.assertRaises(AssertionError): 

468 self.catalog.getChildren(0) 

469 self.catalog.sort(parentKey) 

470 self.catalog.getChildren(0) # Just care this succeeds 

471 

472 def testFitsReadVersion0Compatibility(self): 

473 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data/empty-v0.fits")) 

474 self.assertTrue(cat.getPsfFluxSlot().isValid()) 

475 self.assertTrue(cat.getApFluxSlot().isValid()) 

476 self.assertTrue(cat.getGaussianFluxSlot().isValid()) 

477 self.assertTrue(cat.getModelFluxSlot().isValid()) 

478 self.assertTrue(cat.getCentroidSlot().isValid()) 

479 self.assertTrue(cat.getShapeSlot().isValid()) 

480 self.assertEqual(cat.getPsfFluxSlot().getMeasKey(), 

481 cat.schema.find("flux_psf").key) 

482 self.assertEqual(cat.getApFluxSlot().getMeasKey(), 

483 cat.schema.find("flux_sinc").key) 

484 self.assertEqual(cat.getGaussianFluxSlot().getMeasKey(), 

485 cat.schema.find("flux_naive").key) 

486 self.assertEqual(cat.getModelFluxSlot().getMeasKey(), 

487 cat.schema.find("cmodel_flux").key) 

488 self.assertEqual(cat.getCentroidSlot().getMeasKey().getX(), 

489 cat.schema.find("centroid_sdss_x").key) 

490 self.assertEqual(cat.getCentroidSlot().getMeasKey().getY(), 

491 cat.schema.find("centroid_sdss_y").key) 

492 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxx(), 

493 cat.schema.find("shape_hsm_moments_xx").key) 

494 self.assertEqual(cat.getShapeSlot().getMeasKey().getIyy(), 

495 cat.schema.find("shape_hsm_moments_yy").key) 

496 self.assertEqual(cat.getShapeSlot().getMeasKey().getIxy(), 

497 cat.schema.find("shape_hsm_moments_xy").key) 

498 self.assertEqual(cat.getPsfFluxSlot().getErrKey(), 

499 cat.schema.find("flux_psf_err").key) 

500 self.assertEqual(cat.getApFluxSlot().getErrKey(), 

501 cat.schema.find("flux_sinc_err").key) 

502 self.assertEqual(cat.getGaussianFluxSlot().getErrKey(), 

503 cat.schema.find("flux_naive_err").key) 

504 self.assertEqual(cat.getModelFluxSlot().getErrKey(), 

505 cat.schema.find("cmodel_flux_err").key) 

506 self.assertEqual( 

507 cat.getCentroidSlot().getErrKey(), 

508 lsst.afw.table.CovarianceMatrix2fKey(cat.schema["centroid_sdss_err"], ["x", "y"])) 

509 self.assertEqual( 

510 cat.getShapeSlot().getErrKey(), 

511 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["shape_hsm_moments_err"], ["xx", "yy", "xy"])) 

512 self.assertEqual(cat.getPsfFluxSlot().getFlagKey(), 

513 cat.schema.find("flux_psf_flags").key) 

514 self.assertEqual(cat.getApFluxSlot().getFlagKey(), 

515 cat.schema.find("flux_sinc_flags").key) 

516 self.assertEqual(cat.getGaussianFluxSlot().getFlagKey(), 

517 cat.schema.find("flux_naive_flags").key) 

518 self.assertEqual(cat.getModelFluxSlot().getFlagKey(), 

519 cat.schema.find("cmodel_flux_flags").key) 

520 self.assertEqual(cat.getCentroidSlot().getFlagKey(), 

521 cat.schema.find("centroid_sdss_flags").key) 

522 self.assertEqual(cat.getShapeSlot().getFlagKey(), 

523 cat.schema.find("shape_hsm_moments_flags").key) 

524 

525 def testFitsReadVersion1Compatibility(self): 

526 """Test reading of catalogs with version 1 schema 

527 

528 Version 1 catalogs need to have added aliases from Sigma->Err and 

529 from `_flux`->`_instFlux`. 

530 """ 

531 cat = lsst.afw.table.SourceCatalog.readFits( 

532 os.path.join(testPath, "data", "sourceTable-v1.fits")) 

533 self.assertEqual( 

534 cat.getCentroidSlot().getErrKey(), 

535 lsst.afw.table.CovarianceMatrix2fKey( 

536 cat.schema["slot_Centroid"], 

537 ["x", "y"])) 

538 self.assertEqual( 

539 cat.getShapeSlot().getErrKey(), 

540 lsst.afw.table.CovarianceMatrix3fKey(cat.schema["slot_Shape"], ["xx", "yy", "xy"])) 

541 # check the flux->instFlux conversion 

542 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

543 self.assertEqual(cat.schema["a_fluxSigma"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

544 

545 def testFitsReadVersion2CompatibilityRealSourceCatalog(self): 

546 """DM-15891: some fields were getting aliases they shouldn't have.""" 

547 cat = lsst.afw.table.SourceCatalog.readFits( 

548 os.path.join(testPath, "data", "sourceCatalog-hsc-v2.fits")) 

549 self.assertNotIn('base_SdssShape_flux_xxinstFlux', cat.schema) 

550 self.assertIn('base_SdssShape_instFlux_xx_Cov', cat.schema) 

551 self.assertNotIn('base_Blendedness_abs_flux_cinstFlux', cat.schema) 

552 

553 def testFitsReadVersion2CompatibilityRealCoaddMeasCatalog(self): 

554 """DM-16068: some fields were not getting aliases they should have 

555 

556 In particular, the alias setting relies on flux fields having their 

557 units set properly. Prior to a resolution of DM-16068, the units 

558 were not getting set for several CModel flux fields and one deblender 

559 field (deblend_psfFlux), and thus were not getting the 

560 `_flux`->`_instFlux` aliases set. 

561 

562 NOTE: this test will (is meant to) fail on the read until DM-16068 is 

563 resolved. The error is: 

564 

565 lsst::pex::exceptions::NotFoundError: 'Field or subfield with 

566 name 'modelfit_CModel_instFlux' not found with type 'D'.' 

567 """ 

568 cat = lsst.afw.table.SourceCatalog.readFits( 

569 os.path.join(testPath, "data", "deepCoadd_meas_HSC_v2.fits")) 

570 self.assertIn('modelfit_CModel_instFlux', cat.schema) 

571 self.assertIn('modelfit_CModel_instFluxErr', cat.schema) 

572 self.assertIn('modelfit_CModel_instFlux_inner', cat.schema) 

573 self.assertIn('modelfit_CModel_dev_instFlux_inner', cat.schema) 

574 self.assertIn('modelfit_CModel_exp_instFlux_inner', cat.schema) 

575 self.assertIn('modelfit_CModel_initial_instFlux_inner', cat.schema) 

576 

577 def testFitsVersion2Compatibility(self): 

578 """Test reading of catalogs with version 2 schema 

579 

580 Version 2 catalogs need to have added aliases from `_flux`->`_instFlux`. 

581 """ 

582 cat = lsst.afw.table.SourceCatalog.readFits(os.path.join(testPath, "data", "sourceTable-v2.fits")) 

583 # check the flux->instFlux conversion 

584 self.assertEqual(cat.schema["a_flux"].asKey(), cat.schema["a_instFlux"].asKey()) 

585 self.assertEqual(cat.schema["a_fluxErr"].asKey(), cat.schema["a_instFluxErr"].asKey()) 

586 

587 def testDM1083(self): 

588 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

589 st = lsst.afw.table.SourceTable.make(schema) 

590 cat = lsst.afw.table.SourceCatalog(st) 

591 tmp = lsst.afw.table.SourceCatalog(cat.getTable()) 

592 record = tmp.addNew() 

593 cat.extend(tmp) 

594 self.assertEqual(cat[0].getId(), record.getId()) 

595 # check that the same record is in both catalogs (not a copy) 

596 record.setId(15) 

597 self.assertEqual(cat[0].getId(), record.getId()) 

598 

599 def testSlotUndefine(self): 

600 """Test that we can correctly define and undefine a slot after a SourceTable has been created""" 

601 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

602 key = schema.addField("a_instFlux", type=np.float64, doc="flux field") 

603 table = lsst.afw.table.SourceTable.make(schema) 

604 table.definePsfFlux("a") 

605 self.assertEqual(table.getPsfFluxSlot().getMeasKey(), key) 

606 table.schema.getAliasMap().erase("slot_PsfFlux") 

607 self.assertFalse(table.getPsfFluxSlot().isValid()) 

608 

609 def testOldFootprintPersistence(self): 

610 """Test that we can still read SourceCatalogs with (Heavy)Footprints saved by an older 

611 version of the pipeline with a different format. 

612 """ 

613 filename = os.path.join(testPath, "data", "old-footprint-persistence.fits") 

614 catalog1 = lsst.afw.table.SourceCatalog.readFits(filename) 

615 self.assertEqual(len(catalog1), 2) 

616 with self.assertRaises(LookupError): 

617 catalog1.schema.find("footprint") 

618 fp1 = catalog1[0].getFootprint() 

619 fp2 = catalog1[1].getFootprint() 

620 self.assertEqual(fp1.getArea(), 495) 

621 self.assertEqual(fp2.getArea(), 767) 

622 self.assertFalse(fp1.isHeavy()) 

623 self.assertTrue(fp2.isHeavy()) 

624 self.assertEqual(len(fp1.getSpans()), 29) 

625 self.assertEqual(len(fp2.getSpans()), 44) 

626 self.assertEqual(len(fp1.getPeaks()), 1) 

627 self.assertEqual(len(fp2.getPeaks()), 1) 

628 self.assertEqual(fp1.getBBox(), 

629 lsst.geom.Box2I(lsst.geom.Point2I(129, 2), lsst.geom.Extent2I(25, 29))) 

630 self.assertEqual(fp2.getBBox(), 

631 lsst.geom.Box2I(lsst.geom.Point2I(1184, 2), lsst.geom.Extent2I(78, 38))) 

632 hfp = lsst.afw.detection.HeavyFootprintF(fp2) 

633 self.assertEqual(len(hfp.getImageArray()), fp2.getArea()) 

634 self.assertEqual(len(hfp.getMaskArray()), fp2.getArea()) 

635 self.assertEqual(len(hfp.getVarianceArray()), fp2.getArea()) 

636 catalog2 = lsst.afw.table.SourceCatalog.readFits( 

637 filename, flags=lsst.afw.table.SOURCE_IO_NO_HEAVY_FOOTPRINTS) 

638 self.assertEqual(list(fp1.getSpans()), 

639 list(catalog2[0].getFootprint().getSpans())) 

640 self.assertEqual(list(fp2.getSpans()), 

641 list(catalog2[1].getFootprint().getSpans())) 

642 self.assertFalse(catalog2[1].getFootprint().isHeavy()) 

643 catalog3 = lsst.afw.table.SourceCatalog.readFits( 

644 filename, flags=lsst.afw.table.SOURCE_IO_NO_FOOTPRINTS) 

645 self.assertEqual(catalog3[0].getFootprint(), None) 

646 self.assertEqual(catalog3[1].getFootprint(), None) 

647 

648 def _testFluxSlot(self, slotName): 

649 """Demonstrate that we can create & use the named Flux slot.""" 

650 schema = lsst.afw.table.SourceTable.makeMinimalSchema() 

651 baseName = "afw_Test" 

652 instFluxKey = schema.addField(f"{baseName}_instFlux", type=np.float64, doc="flux") 

653 errKey = schema.addField(f"{baseName}_instFluxErr", type=np.float64, doc="flux uncertainty") 

654 flagKey = schema.addField(f"{baseName}_flag", type="Flag", doc="flux flag") 

655 catalog = lsst.afw.table.SourceCatalog(schema) 

656 table = catalog.table 

657 

658 # Initially, the slot is undefined. 

659 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

660 

661 # After definition, it maps to the keys defined above. 

662 getattr(table, f"define{slotName}")(baseName) 

663 self.assertTrue(getattr(table, f"get{slotName}Slot")().isValid()) 

664 self.assertEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

665 self.assertEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

666 self.assertEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

667 

668 # We should be able to retrieve arbitrary values set in records. 

669 record = catalog.addNew() 

670 instFlux, err, flag = 10.0, 1.0, False 

671 record.set(instFluxKey, instFlux) 

672 record.set(errKey, err) 

673 record.set(flagKey, flag) 

674 instFluxName = slotName.replace("Flux", "InstFlux") 

675 self.assertEqual(getattr(record, f"get{instFluxName}")(), instFlux) 

676 self.assertEqual(getattr(record, f"get{instFluxName}Err")(), err) 

677 self.assertEqual(getattr(record, f"get{slotName}Flag")(), flag) 

678 

679 # Should also be able to retrieve them as columns from the catalog 

680 self.assertEqual(getattr(catalog, f"get{instFluxName}")()[0], instFlux) 

681 self.assertEqual(getattr(catalog, f"get{instFluxName}Err")()[0], err) 

682 

683 # And we should be able to delete the slot, breaking the mapping. 

684 table.schema.getAliasMap().erase(f"slot_{slotName}") 

685 self.assertFalse(getattr(table, f"get{slotName}Slot")().isValid()) 

686 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getMeasKey(), instFluxKey) 

687 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getErrKey(), errKey) 

688 self.assertNotEqual(getattr(table, f"get{slotName}Slot")().getFlagKey(), flagKey) 

689 

690 # When the slot has been deleted, attempting to access it should 

691 # throw a LogicError. 

692 with self.assertRaises(lsst.pex.exceptions.LogicError): 

693 getattr(catalog, f"get{instFluxName}")() 

694 with self.assertRaises(lsst.pex.exceptions.LogicError): 

695 getattr(catalog, f"get{instFluxName}Err")() 

696 

697 def testFluxSlots(self): 

698 """Check that all the expected flux slots are present & correct.""" 

699 for slotName in ["ApFlux", "CalibFlux", "GaussianFlux", "ModelFlux", 

700 "PsfFlux"]: 

701 self._testFluxSlot(slotName) 

702 

703 # But, of course, we should not accept a slot which hasn't be defined. 

704 with self.assertRaises(AttributeError): 

705 self._testFluxSlot("NotExtantFlux") 

706 

707 def testStr(self): 

708 """Check that the str() produced on a catalog contains expected things.""" 

709 string = str(self.catalog) 

710 for field in ('id', 'coord_ra', 'coord_dec'): 

711 self.assertIn(field, string) 

712 

713 def testRepr(self): 

714 """Check that the repr() produced on a catalog contains expected things.""" 

715 string = repr(self.catalog) 

716 self.assertIn(str(type(self.catalog)), string) 

717 for field in ('id', 'coord_ra', 'coord_dec'): 

718 self.assertIn(field, string) 

719 

720 def testStrNonContiguous(self): 

721 """Check that str() doesn't fail on non-contiguous tables.""" 

722 del self.catalog[1] 

723 string = str(self.catalog) 

724 self.assertIn('Non-contiguous afw.Catalog of 2 rows.', string) 

725 for field in ('id', 'coord_ra', 'coord_dec'): 

726 self.assertIn(field, string) 

727 

728 def testRecordStr(self): 

729 """Test that str(record) contains expected things.""" 

730 string = str(self.catalog[0]) 

731 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

732 self.assertIn(field, string) 

733 

734 def testRecordRepr(self): 

735 """Test that repr(record) contains expected things.""" 

736 string = repr(self.catalog[0]) 

737 self.assertIn(str(type(self.catalog[0])), string) 

738 for field in ('id: 50', 'coord_ra: nan', 'coord_dec: nan'): 

739 self.assertIn(field, string) 

740 

741 def testGetNonContiguous(self): 

742 """Check that we can index on non-contiguous tables""" 

743 # Make a non-contiguous catalog 

744 nonContiguous = type(self.catalog)(self.catalog.table) 

745 for rr in reversed(self.catalog): 

746 nonContiguous.append(rr) 

747 num = len(self.catalog) 

748 # Check assumptions 

749 self.assertFalse(nonContiguous.isContiguous()) # We managed to produce a non-contiguous catalog 

750 self.assertEqual(len(set(self.catalog["id"])), num) # ID values are unique 

751 # Indexing with boolean array 

752 select = np.zeros(num, dtype=bool) 

753 select[1] = True 

754 self.assertEqual(nonContiguous[np.flip(select, 0)]["id"], self.catalog[select]["id"]) 

755 # Extracting a number column 

756 column = "a_instFlux" 

757 array = nonContiguous[column] 

758 self.assertFloatsEqual(np.flip(array, 0), self.catalog[column]) 

759 with self.assertRaises(ValueError): 

760 array[1] = 1.2345 # Should be immutable 

761 # Extracting a flag column 

762 column = "a_flag" 

763 array = nonContiguous[column] 

764 np.testing.assert_equal(np.flip(array, 0), self.catalog[column]) 

765 with self.assertRaises(ValueError): 

766 array[1] = True # Should be immutable 

767 

768 

769class MemoryTester(lsst.utils.tests.MemoryTestCase): 

770 pass 

771 

772 

773def setup_module(module): 

774 lsst.utils.tests.init() 

775 

776 

777if __name__ == "__main__": 777 ↛ 778line 777 didn't jump to line 778, because the condition on line 777 was never true

778 lsst.utils.tests.init() 

779 unittest.main()