Coverage for tests/test_simpleTable.py: 6%

668 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-16 03:19 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import os.path 

23import unittest 

24 

25import numpy as np 

26 

27import lsst.utils.tests 

28import lsst.pex.exceptions 

29import lsst.geom 

30import lsst.daf.base 

31import lsst.afw.table 

32import lsst.afw.fits 

33 

34# Testing files live under this, in `data/`. 

35testPath = os.path.abspath(os.path.dirname(__file__)) 

36 

37 

38def makeArray(size, dtype): 

39 return np.array(np.random.randn(size), dtype=dtype) 

40 

41 

42def makeCov(size, dtype): 

43 m = np.array(np.random.randn(size, size), dtype=dtype) 

44 # not quite symmetric for single-precision on some platforms 

45 r = np.dot(m, m.transpose()) 

46 for i in range(r.shape[0]): 

47 for j in range(i): 

48 r[i, j] = r[j, i] 

49 return r 

50 

51 

52class SimpleTableTestCase(lsst.utils.tests.TestCase): 

53 

54 def setUp(self): 

55 np.random.seed(1) 

56 

57 def checkScalarAccessors(self, record, key, name, value1, value2): 

58 fastSetter = getattr(record, "set" + key.getTypeString()) 

59 fastGetter = getattr(record, "get") 

60 record[key] = value1 

61 self.assertEqual(record[key], value1) 

62 self.assertEqual(record.get(key), value1) 

63 self.assertEqual(record[name], value1) 

64 self.assertEqual(record.get(name), value1) 

65 self.assertEqual(fastGetter(key), value1) 

66 record.set(key, value2) 

67 self.assertEqual(record[key], value2) 

68 self.assertEqual(record.get(key), value2) 

69 self.assertEqual(record[name], value2) 

70 self.assertEqual(record.get(name), value2) 

71 self.assertEqual(fastGetter(key), value2) 

72 record[name] = value1 

73 self.assertEqual(record[key], value1) 

74 self.assertEqual(record.get(key), value1) 

75 self.assertEqual(record[name], value1) 

76 self.assertEqual(record.get(name), value1) 

77 self.assertEqual(fastGetter(key), value1) 

78 record.set(name, value2) 

79 self.assertEqual(record[key], value2) 

80 self.assertEqual(record.get(key), value2) 

81 self.assertEqual(record[name], value2) 

82 self.assertEqual(record.get(name), value2) 

83 self.assertEqual(fastGetter(key), value2) 

84 fastSetter(key, value1) 

85 self.assertEqual(record[key], value1) 

86 self.assertEqual(record.get(key), value1) 

87 self.assertEqual(record[name], value1) 

88 self.assertEqual(record.get(name), value1) 

89 self.assertEqual(fastGetter(key), value1) 

90 self.assertIsNone(key.subfields) 

91 

92 def checkArrayAccessors(self, record, key, name, value): 

93 fastSetter = getattr(record, "set" + key.getTypeString()) 

94 fastGetter = getattr(record, "get") 

95 record.set(key, value) 

96 self.assertFloatsEqual(record.get(key), value) 

97 record.set(name, value) 

98 self.assertFloatsEqual(record.get(name), value) 

99 fastSetter(key, value) 

100 self.assertFloatsEqual(fastGetter(key), value) 

101 

102 def testRecordAccess(self): 

103 schema = lsst.afw.table.Schema() 

104 kB = schema.addField("fB", type="B") 

105 kU = schema.addField("fU", type="U") 

106 kI = schema.addField("fI", type="I") 

107 kL = schema.addField("fL", type="L") 

108 kF = schema.addField("fF", type="F") 

109 kD = schema.addField("fD", type="D") 

110 kAngle = schema.addField("fAngle", type="Angle") 

111 kString = schema.addField("fString", type="String", size=4) 

112 kArrayB = schema.addField("fArrayB", type="ArrayB", size=6) 

113 kArrayU = schema.addField("fArrayU", type="ArrayU", size=2) 

114 kArrayI = schema.addField("fArrayI", type="ArrayI", size=3) 

115 kArrayF = schema.addField("fArrayF", type="ArrayF", size=4) 

116 kArrayD = schema.addField("fArrayD", type="ArrayD", size=5) 

117 table = lsst.afw.table.BaseTable.make(schema) 

118 record = table.makeRecord() 

119 self.assertEqual(record[kB], 0) 

120 self.assertEqual(record[kU], 0) 

121 self.assertEqual(record[kI], 0) 

122 self.assertEqual(record[kL], 0) 

123 self.assertTrue(np.isnan(record[kF])) 

124 self.assertTrue(np.isnan(record[kD])) 

125 self.checkScalarAccessors(record, kB, "fB", 4, 5) 

126 self.checkScalarAccessors(record, kU, "fU", 5, 6) 

127 self.checkScalarAccessors(record, kI, "fI", 2, 3) 

128 self.checkScalarAccessors(record, kL, "fL", 2, 3) 

129 self.checkScalarAccessors(record, kF, "fF", 2.5, 3.5) 

130 self.checkScalarAccessors(record, kD, "fD", 2.5, 3.5) 

131 self.checkScalarAccessors(record, kAngle, "fAngle", 

132 5.1*lsst.geom.degrees, -4.1*lsst.geom.degrees) 

133 self.checkScalarAccessors(record, kString, "fString", "ab", "abcd") 

134 self.checkArrayAccessors(record, kArrayB, "fArrayB", 

135 makeArray(kArrayB.getSize(), dtype=np.uint8)) 

136 self.checkArrayAccessors(record, kArrayU, "fArrayU", 

137 makeArray(kArrayU.getSize(), dtype=np.uint16)) 

138 self.checkArrayAccessors(record, kArrayI, "fArrayI", 

139 makeArray(kArrayI.getSize(), dtype=np.int32)) 

140 self.checkArrayAccessors(record, kArrayF, "fArrayF", 

141 makeArray(kArrayF.getSize(), dtype=np.float32)) 

142 self.checkArrayAccessors(record, kArrayD, "fArrayD", 

143 makeArray(kArrayD.getSize(), dtype=np.float64)) 

144 for k in (kArrayF, kArrayD): 

145 self.assertEqual(k.subfields, tuple(range(k.getSize()))) 

146 sub1 = kArrayD.slice(1, 3) 

147 sub2 = kArrayD[0:2] 

148 self.assertFloatsAlmostEqual(record.get(sub1), 

149 record.get(kArrayD)[1:3], rtol=0, atol=0) 

150 self.assertFloatsAlmostEqual(record.get(sub2), 

151 record.get(kArrayD)[0:2], rtol=0, atol=0) 

152 self.assertEqual(sub1[0], sub2[1]) 

153 self.assertIsNone(kAngle.subfields) 

154 k0a = lsst.afw.table.Key["D"]() 

155 k0b = lsst.afw.table.Key["Flag"]() 

156 with self.assertRaises(lsst.pex.exceptions.LogicError): 

157 record.get(k0a) 

158 with self.assertRaises(lsst.pex.exceptions.LogicError): 

159 record.get(k0b) 

160 

161 def _testBaseFits(self, target): 

162 schema = lsst.afw.table.Schema() 

163 k = schema.addField("f", type="D") 

164 cat1 = lsst.afw.table.BaseCatalog(schema) 

165 for i in range(50): 

166 record = cat1.addNew() 

167 record.set(k, np.random.randn()) 

168 cat1.writeFits(target) 

169 cat2 = lsst.afw.table.BaseCatalog.readFits(target) 

170 self.assertEqual(len(cat1), len(cat2)) 

171 for r1, r2 in zip(cat1, cat2): 

172 self.assertEqual(r1.get(k), r2.get(k)) 

173 

174 def testBaseFits(self): 

175 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 

176 self._testBaseFits(tmpFile) 

177 with self.assertRaises(Exception): 

178 lsst.afw.table.BaseCatalog.readFits("nonexistentfile.fits") 

179 

180 def testMemoryFits(self): 

181 mem = lsst.afw.fits.MemFileManager() 

182 self._testBaseFits(mem) 

183 

184 def testColumnView(self): 

185 schema = lsst.afw.table.Schema() 

186 kB = schema.addField("fB", type="B") 

187 kU = schema.addField("fU", type="U") 

188 kI = schema.addField("fI", type="I") 

189 kF = schema.addField("fF", type="F") 

190 kD = schema.addField("fD", type="D") 

191 kArrayF = schema.addField("fArrayF", type="ArrayF", size=2) 

192 kArrayD = schema.addField("fArrayD", type="ArrayD", size=3) 

193 kAngle = schema.addField("fAngle", type="Angle") 

194 kArrayU = schema.addField("fArrayU", type="ArrayU", size=4) 

195 catalog = lsst.afw.table.BaseCatalog(schema) 

196 catalog.addNew() 

197 catalog[0].set(kB, 5) 

198 catalog[0].set(kU, 1) 

199 catalog[0].set(kI, 2) 

200 catalog[0].set(kF, 0.5) 

201 catalog[0].set(kD, 0.25) 

202 catalog[0].set(kArrayF, np.array([-0.5, -0.25], dtype=np.float32)) 

203 catalog[0].set(kArrayD, np.array([-1.5, -1.25, 3.375], dtype=np.float64)) 

204 catalog[0].set(kAngle, lsst.geom.Angle(0.25)) 

205 catalog[0].set(kArrayU, np.array([2, 3, 4, 1], dtype=np.uint16)) 

206 col1a = catalog[kI] 

207 self.assertEqual(col1a.shape, (1,)) 

208 catalog.addNew() 

209 catalog[1].set(kB, 6) 

210 catalog[1].set(kU, 4) 

211 catalog[1].set(kI, 3) 

212 catalog[1].set(kF, 2.5) 

213 catalog[1].set(kD, 0.75) 

214 catalog[1].set(kArrayF, np.array([-3.25, -0.75], dtype=np.float32)) 

215 catalog[1].set(kArrayD, np.array([-1.25, -2.75, 0.625], dtype=np.float64)) 

216 catalog[1].set(kAngle, lsst.geom.Angle(0.15)) 

217 catalog[1].set(kArrayU, np.array([5, 6, 8, 7], dtype=np.uint16)) 

218 col1b = catalog[kI] 

219 self.assertEqual(col1b.shape, (2,)) 

220 columns = catalog.getColumnView() 

221 for key in [kB, kU, kI, kF, kD]: 

222 array = columns[key] 

223 for i in [0, 1]: 

224 self.assertEqual(array[i], catalog[i].get(key)) 

225 for key in [kArrayF, kArrayD, kArrayU]: 

226 array = columns[key] 

227 for i in [0, 1]: 

228 self.assertFloatsEqual(array[i], catalog[i].get(key)) 

229 for key in [kAngle]: 

230 array = columns[key] 

231 for i in [0, 1]: 

232 self.assertEqual(lsst.geom.Angle(array[i]), 

233 catalog[i].get(key)) 

234 for key in [kB, kU, kI, kF, kD]: 

235 vals = columns[key].copy() 

236 vals *= 2 

237 array = columns[key] 

238 array *= 2 

239 for i in [0, 1]: 

240 self.assertEqual(catalog[i].get(key), vals[i]) 

241 self.assertEqual(array[i], vals[i]) 

242 catalog[kI] = 4 

243 f3v = np.random.randn(2) 

244 catalog["fD"] = f3v 

245 for i in [0, 1]: 

246 self.assertEqual(catalog[i].get(kI), 4) 

247 self.assertEqual(catalog[i].get(kD), f3v[i]) 

248 

249 # Accessing an invalid key should raise. 

250 for keyType in ["Angle", "ArrayB", "ArrayD", "ArrayF", "ArrayI", 

251 "ArrayU", "B", "D", "F", "I", "L", "U"]: 

252 # Default-constructed key is invalid 

253 invalidKey = getattr(lsst.afw.table, f"Key{keyType}")() 

254 self.assertFalse(invalidKey.isValid()) 

255 with self.assertRaises(lsst.pex.exceptions.LogicError): 

256 catalog.get(invalidKey) 

257 

258 def testUnsignedFitsPersistence(self): 

259 """Test FITS round-trip of unsigned short ints, since FITS handles unsigned columns differently 

260 from signed columns. 

261 """ 

262 schema = lsst.afw.table.Schema() 

263 k1 = schema.addField("f1", type=np.uint16, doc="scalar uint16") 

264 k2 = schema.addField("f2", type="ArrayU", doc="array uint16", size=4) 

265 cat1 = lsst.afw.table.BaseCatalog(schema) 

266 record1 = cat1.addNew() 

267 record1.set(k1, 4) 

268 record1.set(k2, np.array([5, 6, 7, 8], dtype=np.uint16)) 

269 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

270 cat1.writeFits(filename) 

271 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

272 record2 = cat2[0] 

273 self.assertEqual(cat1.schema, cat2.schema) 

274 self.assertEqual(record1.get(k1), record2.get(k1)) 

275 self.assertFloatsEqual(record1.get(k2), record2.get(k2)) 

276 

277 def testIteration(self): 

278 schema = lsst.afw.table.Schema() 

279 k = schema.addField("a", type=np.int32) 

280 catalog = lsst.afw.table.BaseCatalog(schema) 

281 for n in range(5): 

282 record = catalog.addNew() 

283 record[k] = n 

284 for n, r in enumerate(catalog): 

285 self.assertEqual(n, r[k]) 

286 

287 def testTicket2262(self): 

288 """Test that we can construct an array field in Python. 

289 """ 

290 f1 = lsst.afw.table.Field["ArrayF"]("name", "doc", "barn", size=5) 

291 f2 = lsst.afw.table.Field["ArrayD"]("name", "doc", size=5) 

292 self.assertEqual(f1.getSize(), 5) 

293 self.assertEqual(f2.getSize(), 5) 

294 

295 def testExtract(self): 

296 schema = lsst.afw.table.Schema() 

297 schema.addField("a_b_c1", type=np.float64) 

298 schema.addField("a_b_c2", type="Flag") 

299 schema.addField("a_d1", type=np.int32) 

300 schema.addField("a_d2", type=np.float32) 

301 pointKey = lsst.afw.table.Point2IKey.addFields( 

302 schema, "q_e1", "doc for point field", "pixel") 

303 schema.addField("q_e2_xxErr", type=np.float32) 

304 schema.addField("q_e2_yyErr", type=np.float32) 

305 schema.addField("q_e2_xyErr", type=np.float32) 

306 schema.addField("q_e2_xx_yy_Cov", type=np.float32) 

307 schema.addField("q_e2_xx_xy_Cov", type=np.float32) 

308 schema.addField("q_e2_yy_xy_Cov", type=np.float32) 

309 covKey = lsst.afw.table.CovarianceMatrix3fKey( 

310 schema["q_e2"], ["xx", "yy", "xy"]) 

311 self.assertEqual( 

312 list(schema.extract("a_b_*", ordered=True).keys()), ["a_b_c1", "a_b_c2"]) 

313 self.assertEqual( 

314 list(schema.extract("*1", ordered=True).keys()), ["a_b_c1", "a_d1"]) 

315 self.assertEqual(list(schema.extract("a_b_*", "*2", ordered=True).keys()), 

316 ["a_b_c1", "a_b_c2", "a_d2"]) 

317 self.assertEqual(list(schema.extract(regex=r"a_(.+)1", sub=r"\1f", 

318 ordered=True).keys()), ["b_cf", "df"]) 

319 catalog = lsst.afw.table.BaseCatalog(schema) 

320 for i in range(5): 

321 record = catalog.addNew() 

322 record.set("a_b_c1", np.random.randn()) 

323 record.set("a_b_c2", True) 

324 record.set("a_d1", np.random.randint(100)) 

325 record.set("a_d2", np.random.randn()) 

326 record.set(pointKey, 

327 lsst.geom.Point2I(np.random.randint(10), np.random.randint(10))) 

328 record.set(covKey, np.random.randn(3, 3).astype(np.float32)) 

329 d = record.extract("*") 

330 self.assertEqual(set(d.keys()), set(schema.getNames())) 

331 self.assertEqual(d["a_b_c1"], record.get("a_b_c1")) 

332 self.assertEqual(d["a_b_c2"], record.get("a_b_c2")) 

333 self.assertEqual(d["a_d1"], record.get("a_d1")) 

334 self.assertEqual(d["a_d2"], record.get("a_d2")) 

335 self.assertEqual(d["q_e1_x"], record.get(pointKey.getX())) 

336 self.assertEqual(d["q_e1_y"], record.get(pointKey.getY())) 

337 allIdx = slice(None) 

338 sliceIdx = slice(0, 4, 2) 

339 boolIdx = np.array([True, False, False, True, True]) 

340 for kwds, idx in [ 

341 ({}, allIdx), 

342 ({"copy": True}, allIdx), 

343 ({"where": boolIdx}, boolIdx), 

344 ({"where": sliceIdx}, sliceIdx), 

345 ({"where": boolIdx, "copy": True}, boolIdx), 

346 ({"where": sliceIdx, "copy": True}, sliceIdx), 

347 ]: 

348 d = catalog.extract("*", **kwds) 

349 np.testing.assert_array_equal( 

350 d["a_b_c1"], catalog.get("a_b_c1")[idx]) 

351 np.testing.assert_array_equal( 

352 d["a_d1"], catalog.get("a_d1")[idx]) 

353 np.testing.assert_array_equal( 

354 d["a_d2"], catalog.get("a_d2")[idx]) 

355 np.testing.assert_array_equal( 

356 d["q_e1_x"], catalog.get("q_e1_x")[idx]) 

357 np.testing.assert_array_equal( 

358 d["q_e1_y"], catalog.get("q_e1_y")[idx]) 

359 if "copy" in kwds or idx is boolIdx: 

360 for col in d.values(): 

361 self.assertTrue(col.flags.c_contiguous) 

362 # Test that aliases are included in extract() 

363 schema.getAliasMap().set("b_f", "a_b") 

364 d = schema.extract("b_f*") 

365 self.assertEqual(sorted(d.keys()), ["b_f_c1", "b_f_c2"]) 

366 

367 def testExtend(self): 

368 schema1 = lsst.afw.table.SourceTable.makeMinimalSchema() 

369 k1 = schema1.addField("f1", type=np.int32) 

370 k2 = schema1.addField("f2", type=np.float64) 

371 cat1 = lsst.afw.table.BaseCatalog(schema1) 

372 for i in range(1000): 

373 record = cat1.addNew() 

374 record.setI(k1, i) 

375 record.setD(k2, np.random.randn()) 

376 self.assertFalse(cat1.isContiguous()) 

377 cat2 = lsst.afw.table.BaseCatalog(schema1) 

378 cat2.extend(cat1, deep=True) 

379 self.assertEqual(len(cat1), len(cat2)) 

380 self.assertTrue(cat2.isContiguous()) 

381 cat3 = lsst.afw.table.BaseCatalog(cat1.table) 

382 cat3.extend(cat1, deep=False) 

383 self.assertFalse(cat3.isContiguous()) 

384 cat4 = lsst.afw.table.BaseCatalog(cat1.table) 

385 cat4.extend(list(cat1), deep=False) 

386 self.assertFalse(cat4.isContiguous()) 

387 cat4 = lsst.afw.table.BaseCatalog(schema1) 

388 cat4.extend(list(cat1), deep=True) 

389 self.assertFalse(cat4.isContiguous()) 

390 mapper = lsst.afw.table.SchemaMapper(schema1) 

391 mapper.addMinimalSchema(lsst.afw.table.SourceTable.makeMinimalSchema()) 

392 mapper.addMapping(k2) 

393 schema2 = mapper.getOutputSchema() 

394 self.assertTrue(mapper.getOutputSchema().contains( 

395 lsst.afw.table.SourceTable.makeMinimalSchema())) 

396 cat5 = lsst.afw.table.BaseCatalog(schema2) 

397 cat5.extend(cat1, mapper=mapper) 

398 self.assertTrue(cat5.isContiguous()) 

399 cat6 = lsst.afw.table.SourceCatalog(schema2) 

400 cat6.extend(list(cat1), mapper=mapper) 

401 self.assertFalse(cat6.isContiguous()) 

402 cat7 = lsst.afw.table.SourceCatalog(schema2) 

403 cat7.reserve(len(cat1)*3) 

404 cat7.extend(list(cat1), mapper=mapper) 

405 cat7.extend(cat1, mapper) 

406 cat7.extend(list(cat1), mapper) 

407 self.assertTrue(cat7.isContiguous()) 

408 cat8 = lsst.afw.table.BaseCatalog(schema2) 

409 cat8.extend(list(cat7), True) 

410 cat8.extend(list(cat7), deep=True) 

411 

412 def testTicket2308(self): 

413 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

414 mapper1 = lsst.afw.table.SchemaMapper(inputSchema) 

415 mapper1.addMinimalSchema( 

416 lsst.afw.table.SourceTable.makeMinimalSchema(), True) 

417 mapper2 = lsst.afw.table.SchemaMapper(inputSchema) 

418 mapper2.addMinimalSchema( 

419 lsst.afw.table.SourceTable.makeMinimalSchema(), False) 

420 inputTable = lsst.afw.table.SourceTable.make(inputSchema) 

421 inputRecord = inputTable.makeRecord() 

422 inputRecord.set("id", 42) 

423 outputTable1 = lsst.afw.table.SourceTable.make( 

424 mapper1.getOutputSchema()) 

425 outputTable2 = lsst.afw.table.SourceTable.make( 

426 mapper2.getOutputSchema()) 

427 outputRecord1 = outputTable1.makeRecord() 

428 outputRecord2 = outputTable2.makeRecord() 

429 self.assertEqual(outputRecord1.getId(), outputRecord2.getId()) 

430 self.assertNotEqual(outputRecord1.getId(), inputRecord.getId()) 

431 outputRecord1.assign(inputRecord, mapper1) 

432 self.assertEqual(outputRecord1.getId(), inputRecord.getId()) 

433 outputRecord2.assign(inputRecord, mapper2) 

434 self.assertNotEqual(outputRecord2.getId(), inputRecord.getId()) 

435 

436 def testTicket2393(self): 

437 schema = lsst.afw.table.Schema() 

438 k = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

439 item = schema.find("i") 

440 self.assertEqual(k, item.key) 

441 

442 def testTicket2850(self): 

443 schema = lsst.afw.table.Schema() 

444 table = lsst.afw.table.BaseTable.make(schema) 

445 self.assertEqual(table.getBufferSize(), 0) 

446 

447 def testTicket2894(self): 

448 """Test boolean-array indexing of catalogs. 

449 """ 

450 schema = lsst.afw.table.Schema() 

451 key = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

452 cat1 = lsst.afw.table.BaseCatalog(schema) 

453 cat1.addNew().set(key, 1) 

454 cat1.addNew().set(key, 2) 

455 cat1.addNew().set(key, 3) 

456 cat2 = cat1[np.array([True, False, False], dtype=bool)] 

457 self.assertFloatsEqual(cat2[key], np.array([1], dtype=np.int32)) 

458 # records compare using pointer equality 

459 self.assertEqual(cat2[0], cat1[0]) 

460 cat3 = cat1[np.array([True, True, False], dtype=bool)] 

461 self.assertFloatsEqual(cat3[key], np.array([1, 2], dtype=np.int32)) 

462 cat4 = cat1[np.array([True, False, True], dtype=bool)] 

463 self.assertFloatsEqual(cat4.copy(deep=True)[ 

464 key], np.array([1, 3], dtype=np.int32)) 

465 

466 def testTicket2938(self): 

467 """Test heterogenous catalogs that have records from multiple tables. 

468 """ 

469 schema = lsst.afw.table.Schema() 

470 schema.addField("i", type=np.int32, doc="doc for i") 

471 cat = lsst.afw.table.BaseCatalog(schema) 

472 cat.addNew() 

473 t1 = lsst.afw.table.BaseTable.make(schema) 

474 cat.append(t1.makeRecord()) 

475 self.assertEqual(cat[-1].getTable(), t1) 

476 with self.assertRaises(lsst.pex.exceptions.RuntimeError): 

477 cat.getColumnView() 

478 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

479 cat.writeFits(filename) # shouldn't throw 

480 schema.addField("d", type=np.float64, doc="doc for d") 

481 t2 = lsst.afw.table.BaseTable.make(schema) 

482 cat.append(t2.makeRecord()) 

483 with self.assertRaises(lsst.pex.exceptions.LogicError): 

484 cat.writeFits(filename) 

485 

486 def testTicket3056(self): 

487 """Test sorting and sort-based searches of Catalogs. 

488 """ 

489 schema = lsst.afw.table.SimpleTable.makeMinimalSchema() 

490 ki = schema.addField("i", type=np.int32, doc="doc for i") 

491 kl = schema.addField("l", type=np.int64, doc="doc for l") 

492 kf = schema.addField("f", type=np.float64, doc="doc for f") 

493 cat = lsst.afw.table.SimpleCatalog(schema) 

494 for j in range(50, 0, -1): 

495 record = cat.addNew() 

496 record.set(ki, j//10) 

497 record.set(kl, j) 

498 record.set(kf, np.random.randn()) 

499 self.assertFalse(cat.isSorted(ki)) 

500 self.assertFalse(cat.isSorted(kl)) 

501 # sort by unique int64 field, try unique lookups 

502 cat.sort(kl) 

503 self.assertTrue(cat.isSorted(kl)) 

504 r10 = cat.find(10, kl) 

505 self.assertEqual(r10.get(kl), 10) 

506 # sort by probably-unique float field, try unique and range lookups 

507 cat.sort(kf) 

508 self.assertTrue(cat.isSorted(kf)) 

509 r10 = cat.find(10, kf) 

510 # latter case virtually impossible 

511 self.assertTrue(r10 is None or r10.get(kf) == 10.0) 

512 i0 = cat.lower_bound(-0.5, kf) 

513 i1 = cat.upper_bound(0.5, kf) 

514 for i in range(i0, i1): 

515 self.assertGreaterEqual(cat[i].get(kf), -0.5) 

516 self.assertLess(cat[i].get(kf), 0.5) 

517 for r in cat[cat.between(-0.5, 0.5, kf)]: 

518 self.assertGreaterEqual(r.get(kf), -0.5) 

519 self.assertLess(r.get(kf), 0.5) 

520 # sort by nonunique int32 field, try range lookups 

521 cat.sort(ki) 

522 self.assertTrue(cat.isSorted(ki)) 

523 s = cat.equal_range(3, ki) 

524 self.assertTrue(cat[s].isSorted(kf)) # test for stable sort 

525 for r in cat[s]: 

526 self.assertEqual(r.get(ki), 3) 

527 self.assertEqual(s.start, cat.lower_bound(3, ki)) 

528 self.assertEqual(s.stop, cat.upper_bound(3, ki)) 

529 

530 def testRename(self): 

531 """Test field-renaming functionality in Field, SchemaMapper. 

532 """ 

533 field1i = lsst.afw.table.Field[np.int32]("i1", "doc for i", "m") 

534 field2i = field1i.copyRenamed("i2") 

535 self.assertEqual(field1i.getName(), "i1") 

536 self.assertEqual(field2i.getName(), "i2") 

537 self.assertEqual(field1i.getDoc(), field2i.getDoc()) 

538 self.assertEqual(field1i.getUnits(), field2i.getUnits()) 

539 field1a = lsst.afw.table.Field["ArrayF"]("a1", "doc for a", "s", 3) 

540 field2a = field1a.copyRenamed("a2") 

541 self.assertEqual(field1a.getName(), "a1") 

542 self.assertEqual(field2a.getName(), "a2") 

543 self.assertEqual(field1a.getDoc(), field2a.getDoc()) 

544 self.assertEqual(field1a.getUnits(), field2a.getUnits()) 

545 self.assertEqual(field1a.getSize(), field2a.getSize()) 

546 schema1 = lsst.afw.table.Schema() 

547 k1i = schema1.addField(field1i) 

548 k1a = schema1.addField(field1a) 

549 mapper = lsst.afw.table.SchemaMapper(schema1) 

550 k2i = mapper.addMapping(k1i, "i2") 

551 k2a = mapper.addMapping(k1a, "a2") 

552 schema2 = mapper.getOutputSchema() 

553 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

554 self.assertEqual(schema2.find(k2i).field.getName(), "i2") 

555 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

556 self.assertEqual(schema2.find(k2a).field.getName(), "a2") 

557 self.assertEqual(schema1.find(k1i).field.getDoc(), 

558 schema2.find(k2i).field.getDoc()) 

559 self.assertEqual(schema1.find(k1a).field.getDoc(), 

560 schema2.find(k2a).field.getDoc()) 

561 self.assertEqual(schema1.find(k1i).field.getUnits(), 

562 schema2.find(k2i).field.getUnits()) 

563 self.assertEqual(schema1.find(k1a).field.getUnits(), 

564 schema2.find(k2a).field.getUnits()) 

565 self.assertEqual(schema1.find(k1a).field.getSize(), 

566 schema2.find(k2a).field.getSize()) 

567 k3i = mapper.addMapping(k1i, "i3") 

568 k3a = mapper.addMapping(k1a, "a3") 

569 schema3 = mapper.getOutputSchema() 

570 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

571 self.assertEqual(schema3.find(k3i).field.getName(), "i3") 

572 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

573 self.assertEqual(schema3.find(k3a).field.getName(), "a3") 

574 self.assertEqual(schema1.find(k1i).field.getDoc(), 

575 schema3.find(k3i).field.getDoc()) 

576 self.assertEqual(schema1.find(k1a).field.getDoc(), 

577 schema3.find(k3a).field.getDoc()) 

578 self.assertEqual(schema1.find(k1i).field.getUnits(), 

579 schema3.find(k3i).field.getUnits()) 

580 self.assertEqual(schema1.find(k1a).field.getUnits(), 

581 schema3.find(k3a).field.getUnits()) 

582 self.assertEqual(schema1.find(k1a).field.getSize(), 

583 schema3.find(k3a).field.getSize()) 

584 

585 def testTicket3066(self): 

586 """Test the doReplace option on Schema.addField. 

587 """ 

588 schema = lsst.afw.table.Schema() 

589 k1a = schema.addField("f1", doc="f1a", type="I") 

590 k2a = schema.addField("f2", doc="f2a", type="Flag") 

591 k3a = schema.addField("f3", doc="f3a", type="ArrayF", size=4) 

592 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

593 schema.addField("f1", doc="f1b", type="I") 

594 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

595 schema.addField("f2", doc="f2b", type="Flag") 

596 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

597 schema.addField("f1", doc="f1b", type="F") 

598 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

599 schema.addField("f2", doc="f2b", type="F") 

600 with self.assertRaises(lsst.pex.exceptions.TypeError): 

601 schema.addField("f1", doc="f1b", type="F", doReplace=True) 

602 with self.assertRaises(lsst.pex.exceptions.TypeError): 

603 schema.addField("f2", doc="f2b", type="F", doReplace=True) 

604 with self.assertRaises(lsst.pex.exceptions.TypeError): 

605 schema.addField("f3", doc="f3b", type="ArrayF", 

606 size=3, doReplace=True) 

607 k1b = schema.addField("f1", doc="f1b", type="I", doReplace=True) 

608 self.assertEqual(k1a, k1b) 

609 self.assertEqual(schema.find(k1a).field.getDoc(), "f1b") 

610 k2b = schema.addField("f2", doc="f2b", type="Flag", doReplace=True) 

611 self.assertEqual(k2a, k2b) 

612 self.assertEqual(schema.find(k2a).field.getDoc(), "f2b") 

613 k3b = schema.addField( 

614 "f3", doc="f3b", type="ArrayF", size=4, doReplace=True) 

615 self.assertEqual(k3a, k3b) 

616 self.assertEqual(schema.find(k3a).field.getDoc(), "f3b") 

617 

618 def testDM352(self): 

619 filename = os.path.join(os.path.split(__file__)[0], 

620 "data", "great3.fits") 

621 cat = lsst.afw.table.BaseCatalog.readFits(filename) 

622 self.assertEqual(len(cat), 1) 

623 

624 def testDM1710(self): 

625 # Extending without specifying a mapper or a deep argument should not 

626 # raise. 

627 schema = lsst.afw.table.Schema() 

628 cat1 = lsst.afw.table.BaseCatalog(schema) 

629 cat2 = lsst.afw.table.BaseCatalog(schema) 

630 cat1.extend(cat2) 

631 

632 def testVariableLengthArrays(self): 

633 schema = lsst.afw.table.Schema() 

634 kArrayB = schema.addField("fArrayB", doc="uint8", type="ArrayB", size=0) 

635 kArrayU = schema.addField("fArrayU", doc="uint16", type="ArrayU", size=0) 

636 kArrayI = schema.addField("fArrayI", doc="int32", type="ArrayI", size=0) 

637 kArrayF = schema.addField("fArrayF", doc="single-precision", type="ArrayF") 

638 kArrayD = schema.addField("fArrayD", doc="double-precision", type="ArrayD") 

639 kString = schema.addField("fString", doc="string", type="String", size=0) 

640 cat1 = lsst.afw.table.BaseCatalog(schema) 

641 record1 = cat1.addNew() 

642 self.assertEqual(list(record1.get(kArrayB)), []) 

643 self.assertEqual(list(record1.get(kArrayU)), []) 

644 self.assertEqual(list(record1.get(kArrayI)), []) 

645 self.assertEqual(list(record1.get(kArrayF)), []) 

646 self.assertEqual(list(record1.get(kArrayD)), []) 

647 self.assertEqual(record1.get(kString), "") 

648 dataB = np.random.randint(low=3, high=6, size=4).astype(np.uint8) 

649 dataU = np.random.randint(low=3, high=6, size=4).astype(np.uint16) 

650 dataI = np.random.randint(low=3, high=6, size=4).astype(np.int32) 

651 dataF = np.random.randn(5).astype(np.float32) 

652 dataD = np.random.randn(6).astype(np.float64) 

653 dataString = "the\nquick\tbrown\rfox jumps over the lazy dog" 

654 # Test get/set 

655 record1.set(kArrayB, dataB) 

656 record1.set(kArrayU, dataU) 

657 record1.set(kArrayI, dataI) 

658 record1.set(kArrayF, dataF) 

659 record1.set(kArrayD, dataD) 

660 record1.set(kString, dataString) 

661 self.assertFloatsEqual(record1.get(kArrayB), dataB) 

662 self.assertFloatsEqual(record1.get(kArrayU), dataU) 

663 self.assertFloatsEqual(record1.get(kArrayI), dataI) 

664 self.assertFloatsEqual(record1.get(kArrayF), dataF) 

665 self.assertFloatsEqual(record1.get(kArrayD), dataD) 

666 self.assertEqual(record1.get(kString), dataString) 

667 # Test __getitem__ and view semantics 

668 record1[kArrayF][2] = 3.5 

669 self.assertEqual(dataF[2], 3.5) 

670 # Check that we throw when we try to index a variable-length array Key 

671 with self.assertRaises(lsst.pex.exceptions.LogicError): 

672 kArrayI[0] 

673 with self.assertRaises(lsst.pex.exceptions.LogicError): 

674 kArrayI[0:1] 

675 # Test copying records, both with and without SchemaMapper 

676 record2 = cat1.addNew() 

677 record2.assign(record1) 

678 self.assertFloatsEqual(record2.get(kArrayB), dataB) 

679 self.assertFloatsEqual(record2.get(kArrayU), dataU) 

680 self.assertFloatsEqual(record2.get(kArrayI), dataI) 

681 self.assertFloatsEqual(record2.get(kArrayF), dataF) 

682 self.assertFloatsEqual(record2.get(kArrayD), dataD) 

683 self.assertEqual(record2.get(kString), dataString) 

684 record1[kArrayF][2] = 4.5 

685 # copy in assign() should be deep 

686 self.assertEqual(record2[kArrayF][2], 3.5) 

687 mapper = lsst.afw.table.SchemaMapper(schema) 

688 kb2 = mapper.addMapping(kArrayF) 

689 cat2 = lsst.afw.table.BaseCatalog(mapper.getOutputSchema()) 

690 record3 = cat2.addNew() 

691 record3.assign(record1, mapper) 

692 self.assertFloatsEqual(record3.get(kb2), dataF) 

693 # Test that we throw if we try to get a column view of a 

694 # variable-length arry 

695 with self.assertRaises(lsst.pex.exceptions.LogicError): 

696 cat1.get(kArrayI) 

697 # Test that we can round-trip variable-length arrays through FITS 

698 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

699 cat1.writeFits(filename) 

700 cat3 = lsst.afw.table.BaseCatalog.readFits(filename) 

701 self.assertEqual(schema.compare(cat3.schema, lsst.afw.table.Schema.IDENTICAL), 

702 lsst.afw.table.Schema.IDENTICAL) 

703 record4 = cat3[0] 

704 np.testing.assert_array_equal(record4.get(kArrayB), dataB) 

705 np.testing.assert_array_equal(record4.get(kArrayU), dataU) 

706 np.testing.assert_array_equal(record4.get(kArrayI), dataI) 

707 np.testing.assert_array_equal(record4.get(kArrayF), dataF) 

708 np.testing.assert_array_equal(record4.get(kArrayD), dataD) 

709 self.assertEqual(record4.get(kString), dataString) 

710 

711 def testCompoundFieldFitsConversion(self): 

712 """Test that we convert compound fields saved with an older version of the pipeline 

713 into the set of multiple fields used by their replacement FunctorKeys. 

714 """ 

715 geomValues = { 

716 "point_i_x": 4, "point_i_y": 5, 

717 "point_d_x": 3.5, "point_d_y": 2.0, 

718 "moments_xx": 5.0, "moments_yy": 6.5, "moments_xy": 2.25, 

719 "coord_ra": 1.0*lsst.geom.radians, "coord_dec": 0.5*lsst.geom.radians, 

720 } 

721 covValues = { 

722 "cov_z": np.array([[4.00, 1.25, 1.50, 0.75], 

723 [1.25, 2.25, 0.50, 0.25], 

724 [1.50, 0.50, 6.25, 1.75], 

725 [0.75, 0.25, 1.75, 9.00]], dtype=np.float32), 

726 "cov_p": np.array([[5.50, -2.0], 

727 [-2.0, 3.25]], dtype=np.float32), 

728 "cov_m": np.array([[3.75, -0.5, 1.25], 

729 [-0.5, 4.50, 0.75], 

730 [1.25, 0.75, 6.25]], dtype=np.float32), 

731 } 

732 filename = os.path.join(os.path.split(__file__)[0], 

733 "data", "CompoundFieldConversion.fits") 

734 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

735 record2 = cat2[0] 

736 for k, v in geomValues.items(): 

737 self.assertEqual(record2.get(k), v, msg=k) 

738 covZKey = lsst.afw.table.CovarianceMatrixXfKey( 

739 cat2.schema["cov_z"], ["0", "1", "2", "3"]) 

740 covPKey = lsst.afw.table.CovarianceMatrix2fKey( 

741 cat2.schema["cov_p"], ["x", "y"]) 

742 covMKey = lsst.afw.table.CovarianceMatrix3fKey( 

743 cat2.schema["cov_m"], ["xx", "yy", "xy"]) 

744 self.assertFloatsAlmostEqual(record2.get(covZKey), 

745 covValues["cov_z"], rtol=1E-6) 

746 self.assertFloatsAlmostEqual(record2.get(covPKey), 

747 covValues["cov_p"], rtol=1E-6) 

748 self.assertFloatsAlmostEqual(record2.get(covMKey), 

749 covValues["cov_m"], rtol=1E-6) 

750 

751 def testFitsReadVersion1Compatibility(self): 

752 """Test that v1 SimpleCatalogs read from FITS get correct aliases. 

753 """ 

754 filename = os.path.join(testPath, "data", "ps1-refcat-v1.fits") 

755 catalog = lsst.afw.table.SimpleCatalog.readFits(filename) 

756 self.assertIn('g_flux', catalog.schema) 

757 self.assertNotIn('g_instFlux', catalog.schema) 

758 

759 self.assertIn('g_fluxErr', catalog.schema) 

760 self.assertNotIn('g_instFluxErr', catalog.schema) 

761 

762 def testDelete(self): 

763 schema = lsst.afw.table.Schema() 

764 key = schema.addField("a", type=np.float64, doc="doc for 'a'") 

765 catalog = lsst.afw.table.BaseCatalog(schema) 

766 for i in range(10): 

767 catalog.addNew().set(key, i) 

768 del catalog[4] 

769 self.assertEqual(len(catalog), 9) 

770 self.assertEqual([r.get(key) for r in catalog], 

771 [0, 1, 2, 3, 5, 6, 7, 8, 9]) 

772 del catalog[4:7] 

773 self.assertEqual(len(catalog), 6) 

774 self.assertEqual([r.get(key) for r in catalog], 

775 [0, 1, 2, 3, 8, 9]) 

776 with self.assertRaises(IndexError): 

777 del catalog[1:3:-1] 

778 with self.assertRaises(IndexError): 

779 del catalog[:4:2] 

780 with self.assertRaises(IndexError): 

781 del catalog[50] 

782 

783 def testSetFlagColumn(self): 

784 schema = lsst.afw.table.Schema() 

785 key = schema.addField("a", type="Flag", doc="doc for 'a'") 

786 catalog = lsst.afw.table.BaseCatalog(schema) 

787 catalog.resize(5) 

788 # Set scalar with key. 

789 catalog[key] = True 

790 self.assertEqual(list(catalog[key]), [True] * 5) 

791 # Set scalar with name. 

792 catalog["a"] = False 

793 self.assertEqual(list(catalog[key]), [False] * 5) 

794 # Set array with key. 

795 v1 = np.array([True, False, True, False, True], dtype=bool) 

796 catalog[key] = v1 

797 self.assertEqual(list(catalog[key]), list(v1)) 

798 # Set array with name. 

799 v2 = np.array([False, True, False, True, False], dtype=bool) 

800 catalog["a"] = v2 

801 self.assertEqual(list(catalog[key]), list(v2)) 

802 

803 def testAngleColumnArrayAccess(self): 

804 """Test column-array access to Angle columns on both contiguous and 

805 non-contiguous arrays. 

806 """ 

807 schema = lsst.afw.table.Schema() 

808 key = schema.addField("a", type="Angle", doc="doc for a") 

809 catalog = lsst.afw.table.BaseCatalog(schema) 

810 catalog.resize(2) 

811 self.assertTrue(catalog.isContiguous()) 

812 catalog[key] = np.array([3.0, 4.0]) 

813 self.assertFloatsEqual(catalog[key], np.array([3.0, 4.0])) 

814 record = catalog.addNew() 

815 self.assertFalse(catalog.isContiguous()) 

816 record[key] = 5.0 * lsst.geom.radians 

817 self.assertFloatsEqual(catalog[key], np.array([3.0, 4.0, 5.0])) 

818 self.assertFalse(catalog[key].flags.writeable) 

819 

820 def testArrayColumnArrayAccess(self): 

821 """Test column-array access to Array columns on both contiguous and 

822 non-contiguous arrays. 

823 """ 

824 schema = lsst.afw.table.Schema() 

825 key = schema.addField("a", type="ArrayD", doc="doc for a", size=3) 

826 catalog = lsst.afw.table.BaseCatalog(schema) 

827 catalog.resize(2) 

828 self.assertTrue(catalog.isContiguous()) 

829 catalog[key] = np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]) 

830 self.assertFloatsEqual(catalog[key], np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]])) 

831 record = catalog.addNew() 

832 self.assertFalse(catalog.isContiguous()) 

833 record[key] = np.array([9.0, 10.0, 11.0]) 

834 self.assertFloatsEqual(catalog[key], np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0], [9.0, 10.0, 11.0]])) 

835 self.assertFalse(catalog[key].flags.writeable) 

836 

837 def testMetadataProperty(self): 

838 """Test that the metadata property of BaseTable works as expected. 

839 """ 

840 schema = lsst.afw.table.Schema() 

841 table = lsst.afw.table.BaseTable.make(schema) 

842 # BaseTable should not have a metadata property on construction. 

843 self.assertIsNone(table.metadata) 

844 metadata = lsst.daf.base.PropertyList() 

845 metadata["one"] = 1 

846 table.metadata = metadata 

847 self.assertEqual(table.metadata["one"], 1) 

848 

849 # SimpleTable should have an empty metadata property on construction. 

850 schema = lsst.afw.table.SimpleTable.makeMinimalSchema() 

851 table = lsst.afw.table.SimpleTable.make(schema) 

852 self.assertEqual(table.metadata, lsst.daf.base.PropertyList()) 

853 

854 

855class MemoryTester(lsst.utils.tests.MemoryTestCase): 

856 pass 

857 

858 

859def setup_module(module): 

860 lsst.utils.tests.init() 

861 

862 

863if __name__ == "__main__": 863 ↛ 864line 863 didn't jump to line 864, because the condition on line 863 was never true

864 lsst.utils.tests.init() 

865 unittest.main()