Coverage for tests/test_simpleTable.py: 6%

658 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-08 03:13 -0700

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for table.SimpleTable 

24 

25Run with: 

26 python test_simpleTable.py 

27or 

28 pytest test_simpleTable.py 

29""" 

30import os.path 

31import unittest 

32 

33import numpy as np 

34 

35import lsst.utils.tests 

36import lsst.pex.exceptions 

37import lsst.geom 

38import lsst.daf.base 

39import lsst.afw.table 

40import lsst.afw.fits 

41 

42# Testing files live under this, in `data/`. 

43testPath = os.path.abspath(os.path.dirname(__file__)) 

44 

45 

46def makeArray(size, dtype): 

47 return np.array(np.random.randn(size), dtype=dtype) 

48 

49 

50def makeCov(size, dtype): 

51 m = np.array(np.random.randn(size, size), dtype=dtype) 

52 # not quite symmetric for single-precision on some platforms 

53 r = np.dot(m, m.transpose()) 

54 for i in range(r.shape[0]): 

55 for j in range(i): 

56 r[i, j] = r[j, i] 

57 return r 

58 

59 

60class SimpleTableTestCase(lsst.utils.tests.TestCase): 

61 

62 def setUp(self): 

63 np.random.seed(1) 

64 

65 def checkScalarAccessors(self, record, key, name, value1, value2): 

66 fastSetter = getattr(record, "set" + key.getTypeString()) 

67 fastGetter = getattr(record, "get") 

68 record[key] = value1 

69 self.assertEqual(record[key], value1) 

70 self.assertEqual(record.get(key), value1) 

71 self.assertEqual(record[name], value1) 

72 self.assertEqual(record.get(name), value1) 

73 self.assertEqual(fastGetter(key), value1) 

74 record.set(key, value2) 

75 self.assertEqual(record[key], value2) 

76 self.assertEqual(record.get(key), value2) 

77 self.assertEqual(record[name], value2) 

78 self.assertEqual(record.get(name), value2) 

79 self.assertEqual(fastGetter(key), value2) 

80 record[name] = value1 

81 self.assertEqual(record[key], value1) 

82 self.assertEqual(record.get(key), value1) 

83 self.assertEqual(record[name], value1) 

84 self.assertEqual(record.get(name), value1) 

85 self.assertEqual(fastGetter(key), value1) 

86 record.set(name, value2) 

87 self.assertEqual(record[key], value2) 

88 self.assertEqual(record.get(key), value2) 

89 self.assertEqual(record[name], value2) 

90 self.assertEqual(record.get(name), value2) 

91 self.assertEqual(fastGetter(key), value2) 

92 fastSetter(key, value1) 

93 self.assertEqual(record[key], value1) 

94 self.assertEqual(record.get(key), value1) 

95 self.assertEqual(record[name], value1) 

96 self.assertEqual(record.get(name), value1) 

97 self.assertEqual(fastGetter(key), value1) 

98 self.assertIsNone(key.subfields) 

99 

100 def checkArrayAccessors(self, record, key, name, value): 

101 fastSetter = getattr(record, "set" + key.getTypeString()) 

102 fastGetter = getattr(record, "get") 

103 record.set(key, value) 

104 self.assertFloatsEqual(record.get(key), value) 

105 record.set(name, value) 

106 self.assertFloatsEqual(record.get(name), value) 

107 fastSetter(key, value) 

108 self.assertFloatsEqual(fastGetter(key), value) 

109 

110 def testRecordAccess(self): 

111 schema = lsst.afw.table.Schema() 

112 kB = schema.addField("fB", type="B") 

113 kU = schema.addField("fU", type="U") 

114 kI = schema.addField("fI", type="I") 

115 kL = schema.addField("fL", type="L") 

116 kF = schema.addField("fF", type="F") 

117 kD = schema.addField("fD", type="D") 

118 kAngle = schema.addField("fAngle", type="Angle") 

119 kString = schema.addField("fString", type="String", size=4) 

120 kArrayB = schema.addField("fArrayB", type="ArrayB", size=6) 

121 kArrayU = schema.addField("fArrayU", type="ArrayU", size=2) 

122 kArrayI = schema.addField("fArrayI", type="ArrayI", size=3) 

123 kArrayF = schema.addField("fArrayF", type="ArrayF", size=4) 

124 kArrayD = schema.addField("fArrayD", type="ArrayD", size=5) 

125 table = lsst.afw.table.BaseTable.make(schema) 

126 record = table.makeRecord() 

127 self.assertEqual(record[kB], 0) 

128 self.assertEqual(record[kU], 0) 

129 self.assertEqual(record[kI], 0) 

130 self.assertEqual(record[kL], 0) 

131 self.assertTrue(np.isnan(record[kF])) 

132 self.assertTrue(np.isnan(record[kD])) 

133 self.checkScalarAccessors(record, kB, "fB", 4, 5) 

134 self.checkScalarAccessors(record, kU, "fU", 5, 6) 

135 self.checkScalarAccessors(record, kI, "fI", 2, 3) 

136 self.checkScalarAccessors(record, kL, "fL", 2, 3) 

137 self.checkScalarAccessors(record, kF, "fF", 2.5, 3.5) 

138 self.checkScalarAccessors(record, kD, "fD", 2.5, 3.5) 

139 self.checkScalarAccessors(record, kAngle, "fAngle", 

140 5.1*lsst.geom.degrees, -4.1*lsst.geom.degrees) 

141 self.checkScalarAccessors(record, kString, "fString", "ab", "abcd") 

142 self.checkArrayAccessors(record, kArrayB, "fArrayB", 

143 makeArray(kArrayB.getSize(), dtype=np.uint8)) 

144 self.checkArrayAccessors(record, kArrayU, "fArrayU", 

145 makeArray(kArrayU.getSize(), dtype=np.uint16)) 

146 self.checkArrayAccessors(record, kArrayI, "fArrayI", 

147 makeArray(kArrayI.getSize(), dtype=np.int32)) 

148 self.checkArrayAccessors(record, kArrayF, "fArrayF", 

149 makeArray(kArrayF.getSize(), dtype=np.float32)) 

150 self.checkArrayAccessors(record, kArrayD, "fArrayD", 

151 makeArray(kArrayD.getSize(), dtype=np.float64)) 

152 for k in (kArrayF, kArrayD): 

153 self.assertEqual(k.subfields, tuple(range(k.getSize()))) 

154 sub1 = kArrayD.slice(1, 3) 

155 sub2 = kArrayD[0:2] 

156 self.assertFloatsAlmostEqual(record.get(sub1), 

157 record.get(kArrayD)[1:3], rtol=0, atol=0) 

158 self.assertFloatsAlmostEqual(record.get(sub2), 

159 record.get(kArrayD)[0:2], rtol=0, atol=0) 

160 self.assertEqual(sub1[0], sub2[1]) 

161 self.assertIsNone(kAngle.subfields) 

162 k0a = lsst.afw.table.Key["D"]() 

163 k0b = lsst.afw.table.Key["Flag"]() 

164 with self.assertRaises(lsst.pex.exceptions.LogicError): 

165 record.get(k0a) 

166 with self.assertRaises(lsst.pex.exceptions.LogicError): 

167 record.get(k0b) 

168 

169 def _testBaseFits(self, target): 

170 schema = lsst.afw.table.Schema() 

171 k = schema.addField("f", type="D") 

172 cat1 = lsst.afw.table.BaseCatalog(schema) 

173 for i in range(50): 

174 record = cat1.addNew() 

175 record.set(k, np.random.randn()) 

176 cat1.writeFits(target) 

177 cat2 = lsst.afw.table.BaseCatalog.readFits(target) 

178 self.assertEqual(len(cat1), len(cat2)) 

179 for r1, r2 in zip(cat1, cat2): 

180 self.assertEqual(r1.get(k), r2.get(k)) 

181 

182 def testBaseFits(self): 

183 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 

184 self._testBaseFits(tmpFile) 

185 with self.assertRaises(Exception): 

186 lsst.afw.table.BaseCatalog.readFits("nonexistentfile.fits") 

187 

188 def testMemoryFits(self): 

189 mem = lsst.afw.fits.MemFileManager() 

190 self._testBaseFits(mem) 

191 

192 def testColumnView(self): 

193 schema = lsst.afw.table.Schema() 

194 kB = schema.addField("fB", type="B") 

195 kU = schema.addField("fU", type="U") 

196 kI = schema.addField("fI", type="I") 

197 kF = schema.addField("fF", type="F") 

198 kD = schema.addField("fD", type="D") 

199 kArrayF = schema.addField("fArrayF", type="ArrayF", size=2) 

200 kArrayD = schema.addField("fArrayD", type="ArrayD", size=3) 

201 kAngle = schema.addField("fAngle", type="Angle") 

202 kArrayU = schema.addField("fArrayU", type="ArrayU", size=4) 

203 catalog = lsst.afw.table.BaseCatalog(schema) 

204 catalog.addNew() 

205 catalog[0].set(kB, 5) 

206 catalog[0].set(kU, 1) 

207 catalog[0].set(kI, 2) 

208 catalog[0].set(kF, 0.5) 

209 catalog[0].set(kD, 0.25) 

210 catalog[0].set(kArrayF, np.array([-0.5, -0.25], dtype=np.float32)) 

211 catalog[0].set(kArrayD, np.array([-1.5, -1.25, 3.375], dtype=np.float64)) 

212 catalog[0].set(kAngle, lsst.geom.Angle(0.25)) 

213 catalog[0].set(kArrayU, np.array([2, 3, 4, 1], dtype=np.uint16)) 

214 col1a = catalog[kI] 

215 self.assertEqual(col1a.shape, (1,)) 

216 catalog.addNew() 

217 catalog[1].set(kB, 6) 

218 catalog[1].set(kU, 4) 

219 catalog[1].set(kI, 3) 

220 catalog[1].set(kF, 2.5) 

221 catalog[1].set(kD, 0.75) 

222 catalog[1].set(kArrayF, np.array([-3.25, -0.75], dtype=np.float32)) 

223 catalog[1].set(kArrayD, np.array([-1.25, -2.75, 0.625], dtype=np.float64)) 

224 catalog[1].set(kAngle, lsst.geom.Angle(0.15)) 

225 catalog[1].set(kArrayU, np.array([5, 6, 8, 7], dtype=np.uint16)) 

226 col1b = catalog[kI] 

227 self.assertEqual(col1b.shape, (2,)) 

228 columns = catalog.getColumnView() 

229 for key in [kB, kU, kI, kF, kD]: 

230 array = columns[key] 

231 for i in [0, 1]: 

232 self.assertEqual(array[i], catalog[i].get(key)) 

233 for key in [kArrayF, kArrayD, kArrayU]: 

234 array = columns[key] 

235 for i in [0, 1]: 

236 self.assertFloatsEqual(array[i], catalog[i].get(key)) 

237 for key in [kAngle]: 

238 array = columns[key] 

239 for i in [0, 1]: 

240 self.assertEqual(lsst.geom.Angle(array[i]), 

241 catalog[i].get(key)) 

242 for key in [kB, kU, kI, kF, kD]: 

243 vals = columns[key].copy() 

244 vals *= 2 

245 array = columns[key] 

246 array *= 2 

247 for i in [0, 1]: 

248 self.assertEqual(catalog[i].get(key), vals[i]) 

249 self.assertEqual(array[i], vals[i]) 

250 catalog[kI] = 4 

251 f3v = np.random.randn(2) 

252 catalog["fD"] = f3v 

253 for i in [0, 1]: 

254 self.assertEqual(catalog[i].get(kI), 4) 

255 self.assertEqual(catalog[i].get(kD), f3v[i]) 

256 

257 # Accessing an invalid key should raise. 

258 for keyType in ["Angle", "ArrayB", "ArrayD", "ArrayF", "ArrayI", 

259 "ArrayU", "B", "D", "F", "I", "L", "U"]: 

260 # Default-constructed key is invalid 

261 invalidKey = getattr(lsst.afw.table, f"Key{keyType}")() 

262 self.assertFalse(invalidKey.isValid()) 

263 with self.assertRaises(lsst.pex.exceptions.LogicError): 

264 catalog.get(invalidKey) 

265 

266 def testUnsignedFitsPersistence(self): 

267 """Test FITS round-trip of unsigned short ints, since FITS handles unsigned columns differently 

268 from signed columns. 

269 """ 

270 schema = lsst.afw.table.Schema() 

271 k1 = schema.addField("f1", type=np.uint16, doc="scalar uint16") 

272 k2 = schema.addField("f2", type="ArrayU", doc="array uint16", size=4) 

273 cat1 = lsst.afw.table.BaseCatalog(schema) 

274 record1 = cat1.addNew() 

275 record1.set(k1, 4) 

276 record1.set(k2, np.array([5, 6, 7, 8], dtype=np.uint16)) 

277 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

278 cat1.writeFits(filename) 

279 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

280 record2 = cat2[0] 

281 self.assertEqual(cat1.schema, cat2.schema) 

282 self.assertEqual(record1.get(k1), record2.get(k1)) 

283 self.assertFloatsEqual(record1.get(k2), record2.get(k2)) 

284 

285 def testIteration(self): 

286 schema = lsst.afw.table.Schema() 

287 k = schema.addField("a", type=np.int32) 

288 catalog = lsst.afw.table.BaseCatalog(schema) 

289 for n in range(5): 

290 record = catalog.addNew() 

291 record[k] = n 

292 for n, r in enumerate(catalog): 

293 self.assertEqual(n, r[k]) 

294 

295 def testTicket2262(self): 

296 """Test that we can construct an array field in Python. 

297 """ 

298 f1 = lsst.afw.table.Field["ArrayF"]("name", "doc", "barn", size=5) 

299 f2 = lsst.afw.table.Field["ArrayD"]("name", "doc", size=5) 

300 self.assertEqual(f1.getSize(), 5) 

301 self.assertEqual(f2.getSize(), 5) 

302 

303 def testExtract(self): 

304 schema = lsst.afw.table.Schema() 

305 schema.addField("a_b_c1", type=np.float64) 

306 schema.addField("a_b_c2", type="Flag") 

307 schema.addField("a_d1", type=np.int32) 

308 schema.addField("a_d2", type=np.float32) 

309 pointKey = lsst.afw.table.Point2IKey.addFields( 

310 schema, "q_e1", "doc for point field", "pixel") 

311 schema.addField("q_e2_xxErr", type=np.float32) 

312 schema.addField("q_e2_yyErr", type=np.float32) 

313 schema.addField("q_e2_xyErr", type=np.float32) 

314 schema.addField("q_e2_xx_yy_Cov", type=np.float32) 

315 schema.addField("q_e2_xx_xy_Cov", type=np.float32) 

316 schema.addField("q_e2_yy_xy_Cov", type=np.float32) 

317 covKey = lsst.afw.table.CovarianceMatrix3fKey( 

318 schema["q_e2"], ["xx", "yy", "xy"]) 

319 self.assertEqual( 

320 list(schema.extract("a_b_*", ordered=True).keys()), ["a_b_c1", "a_b_c2"]) 

321 self.assertEqual( 

322 list(schema.extract("*1", ordered=True).keys()), ["a_b_c1", "a_d1"]) 

323 self.assertEqual(list(schema.extract("a_b_*", "*2", ordered=True).keys()), 

324 ["a_b_c1", "a_b_c2", "a_d2"]) 

325 self.assertEqual(list(schema.extract(regex=r"a_(.+)1", sub=r"\1f", 

326 ordered=True).keys()), ["b_cf", "df"]) 

327 catalog = lsst.afw.table.BaseCatalog(schema) 

328 for i in range(5): 

329 record = catalog.addNew() 

330 record.set("a_b_c1", np.random.randn()) 

331 record.set("a_b_c2", True) 

332 record.set("a_d1", np.random.randint(100)) 

333 record.set("a_d2", np.random.randn()) 

334 record.set(pointKey, 

335 lsst.geom.Point2I(np.random.randint(10), np.random.randint(10))) 

336 record.set(covKey, np.random.randn(3, 3).astype(np.float32)) 

337 d = record.extract("*") 

338 self.assertEqual(set(d.keys()), set(schema.getNames())) 

339 self.assertEqual(d["a_b_c1"], record.get("a_b_c1")) 

340 self.assertEqual(d["a_b_c2"], record.get("a_b_c2")) 

341 self.assertEqual(d["a_d1"], record.get("a_d1")) 

342 self.assertEqual(d["a_d2"], record.get("a_d2")) 

343 self.assertEqual(d["q_e1_x"], record.get(pointKey.getX())) 

344 self.assertEqual(d["q_e1_y"], record.get(pointKey.getY())) 

345 allIdx = slice(None) 

346 sliceIdx = slice(0, 4, 2) 

347 boolIdx = np.array([True, False, False, True, True]) 

348 for kwds, idx in [ 

349 ({}, allIdx), 

350 ({"copy": True}, allIdx), 

351 ({"where": boolIdx}, boolIdx), 

352 ({"where": sliceIdx}, sliceIdx), 

353 ({"where": boolIdx, "copy": True}, boolIdx), 

354 ({"where": sliceIdx, "copy": True}, sliceIdx), 

355 ]: 

356 d = catalog.extract("*", **kwds) 

357 np.testing.assert_array_equal( 

358 d["a_b_c1"], catalog.get("a_b_c1")[idx]) 

359 np.testing.assert_array_equal( 

360 d["a_d1"], catalog.get("a_d1")[idx]) 

361 np.testing.assert_array_equal( 

362 d["a_d2"], catalog.get("a_d2")[idx]) 

363 np.testing.assert_array_equal( 

364 d["q_e1_x"], catalog.get("q_e1_x")[idx]) 

365 np.testing.assert_array_equal( 

366 d["q_e1_y"], catalog.get("q_e1_y")[idx]) 

367 if "copy" in kwds or idx is boolIdx: 

368 for col in d.values(): 

369 self.assertTrue(col.flags.c_contiguous) 

370 # Test that aliases are included in extract() 

371 schema.getAliasMap().set("b_f", "a_b") 

372 d = schema.extract("b_f*") 

373 self.assertEqual(sorted(d.keys()), ["b_f_c1", "b_f_c2"]) 

374 

375 def testExtend(self): 

376 schema1 = lsst.afw.table.SourceTable.makeMinimalSchema() 

377 k1 = schema1.addField("f1", type=np.int32) 

378 k2 = schema1.addField("f2", type=np.float64) 

379 cat1 = lsst.afw.table.BaseCatalog(schema1) 

380 for i in range(1000): 

381 record = cat1.addNew() 

382 record.setI(k1, i) 

383 record.setD(k2, np.random.randn()) 

384 self.assertFalse(cat1.isContiguous()) 

385 cat2 = lsst.afw.table.BaseCatalog(schema1) 

386 cat2.extend(cat1, deep=True) 

387 self.assertEqual(len(cat1), len(cat2)) 

388 self.assertTrue(cat2.isContiguous()) 

389 cat3 = lsst.afw.table.BaseCatalog(cat1.table) 

390 cat3.extend(cat1, deep=False) 

391 self.assertFalse(cat3.isContiguous()) 

392 cat4 = lsst.afw.table.BaseCatalog(cat1.table) 

393 cat4.extend(list(cat1), deep=False) 

394 self.assertFalse(cat4.isContiguous()) 

395 cat4 = lsst.afw.table.BaseCatalog(schema1) 

396 cat4.extend(list(cat1), deep=True) 

397 self.assertFalse(cat4.isContiguous()) 

398 mapper = lsst.afw.table.SchemaMapper(schema1) 

399 mapper.addMinimalSchema(lsst.afw.table.SourceTable.makeMinimalSchema()) 

400 mapper.addMapping(k2) 

401 schema2 = mapper.getOutputSchema() 

402 self.assertTrue(mapper.getOutputSchema().contains( 

403 lsst.afw.table.SourceTable.makeMinimalSchema())) 

404 cat5 = lsst.afw.table.BaseCatalog(schema2) 

405 cat5.extend(cat1, mapper=mapper) 

406 self.assertTrue(cat5.isContiguous()) 

407 cat6 = lsst.afw.table.SourceCatalog(schema2) 

408 cat6.extend(list(cat1), mapper=mapper) 

409 self.assertFalse(cat6.isContiguous()) 

410 cat7 = lsst.afw.table.SourceCatalog(schema2) 

411 cat7.reserve(len(cat1)*3) 

412 cat7.extend(list(cat1), mapper=mapper) 

413 cat7.extend(cat1, mapper) 

414 cat7.extend(list(cat1), mapper) 

415 self.assertTrue(cat7.isContiguous()) 

416 cat8 = lsst.afw.table.BaseCatalog(schema2) 

417 cat8.extend(list(cat7), True) 

418 cat8.extend(list(cat7), deep=True) 

419 

420 def testTicket2308(self): 

421 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

422 mapper1 = lsst.afw.table.SchemaMapper(inputSchema) 

423 mapper1.addMinimalSchema( 

424 lsst.afw.table.SourceTable.makeMinimalSchema(), True) 

425 mapper2 = lsst.afw.table.SchemaMapper(inputSchema) 

426 mapper2.addMinimalSchema( 

427 lsst.afw.table.SourceTable.makeMinimalSchema(), False) 

428 inputTable = lsst.afw.table.SourceTable.make(inputSchema) 

429 inputRecord = inputTable.makeRecord() 

430 inputRecord.set("id", 42) 

431 outputTable1 = lsst.afw.table.SourceTable.make( 

432 mapper1.getOutputSchema()) 

433 outputTable2 = lsst.afw.table.SourceTable.make( 

434 mapper2.getOutputSchema()) 

435 outputRecord1 = outputTable1.makeRecord() 

436 outputRecord2 = outputTable2.makeRecord() 

437 self.assertEqual(outputRecord1.getId(), outputRecord2.getId()) 

438 self.assertNotEqual(outputRecord1.getId(), inputRecord.getId()) 

439 outputRecord1.assign(inputRecord, mapper1) 

440 self.assertEqual(outputRecord1.getId(), inputRecord.getId()) 

441 outputRecord2.assign(inputRecord, mapper2) 

442 self.assertNotEqual(outputRecord2.getId(), inputRecord.getId()) 

443 

444 def testTicket2393(self): 

445 schema = lsst.afw.table.Schema() 

446 k = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

447 item = schema.find("i") 

448 self.assertEqual(k, item.key) 

449 

450 def testTicket2850(self): 

451 schema = lsst.afw.table.Schema() 

452 table = lsst.afw.table.BaseTable.make(schema) 

453 self.assertEqual(table.getBufferSize(), 0) 

454 

455 def testTicket2894(self): 

456 """Test boolean-array indexing of catalogs. 

457 """ 

458 schema = lsst.afw.table.Schema() 

459 key = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

460 cat1 = lsst.afw.table.BaseCatalog(schema) 

461 cat1.addNew().set(key, 1) 

462 cat1.addNew().set(key, 2) 

463 cat1.addNew().set(key, 3) 

464 cat2 = cat1[np.array([True, False, False], dtype=bool)] 

465 self.assertFloatsEqual(cat2[key], np.array([1], dtype=np.int32)) 

466 # records compare using pointer equality 

467 self.assertEqual(cat2[0], cat1[0]) 

468 cat3 = cat1[np.array([True, True, False], dtype=bool)] 

469 self.assertFloatsEqual(cat3[key], np.array([1, 2], dtype=np.int32)) 

470 cat4 = cat1[np.array([True, False, True], dtype=bool)] 

471 self.assertFloatsEqual(cat4.copy(deep=True)[ 

472 key], np.array([1, 3], dtype=np.int32)) 

473 

474 def testTicket2938(self): 

475 """Test heterogenous catalogs that have records from multiple tables. 

476 """ 

477 schema = lsst.afw.table.Schema() 

478 schema.addField("i", type=np.int32, doc="doc for i") 

479 cat = lsst.afw.table.BaseCatalog(schema) 

480 cat.addNew() 

481 t1 = lsst.afw.table.BaseTable.make(schema) 

482 cat.append(t1.makeRecord()) 

483 self.assertEqual(cat[-1].getTable(), t1) 

484 with self.assertRaises(lsst.pex.exceptions.RuntimeError): 

485 cat.getColumnView() 

486 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

487 cat.writeFits(filename) # shouldn't throw 

488 schema.addField("d", type=np.float64, doc="doc for d") 

489 t2 = lsst.afw.table.BaseTable.make(schema) 

490 cat.append(t2.makeRecord()) 

491 with self.assertRaises(lsst.pex.exceptions.LogicError): 

492 cat.writeFits(filename) 

493 

494 def testTicket3056(self): 

495 """Test sorting and sort-based searches of Catalogs. 

496 """ 

497 schema = lsst.afw.table.SimpleTable.makeMinimalSchema() 

498 ki = schema.addField("i", type=np.int32, doc="doc for i") 

499 kl = schema.addField("l", type=np.int64, doc="doc for l") 

500 kf = schema.addField("f", type=np.float64, doc="doc for f") 

501 cat = lsst.afw.table.SimpleCatalog(schema) 

502 for j in range(50, 0, -1): 

503 record = cat.addNew() 

504 record.set(ki, j//10) 

505 record.set(kl, j) 

506 record.set(kf, np.random.randn()) 

507 self.assertFalse(cat.isSorted(ki)) 

508 self.assertFalse(cat.isSorted(kl)) 

509 # sort by unique int64 field, try unique lookups 

510 cat.sort(kl) 

511 self.assertTrue(cat.isSorted(kl)) 

512 r10 = cat.find(10, kl) 

513 self.assertEqual(r10.get(kl), 10) 

514 # sort by probably-unique float field, try unique and range lookups 

515 cat.sort(kf) 

516 self.assertTrue(cat.isSorted(kf)) 

517 r10 = cat.find(10, kf) 

518 # latter case virtually impossible 

519 self.assertTrue(r10 is None or r10.get(kf) == 10.0) 

520 i0 = cat.lower_bound(-0.5, kf) 

521 i1 = cat.upper_bound(0.5, kf) 

522 for i in range(i0, i1): 

523 self.assertGreaterEqual(cat[i].get(kf), -0.5) 

524 self.assertLess(cat[i].get(kf), 0.5) 

525 for r in cat[cat.between(-0.5, 0.5, kf)]: 

526 self.assertGreaterEqual(r.get(kf), -0.5) 

527 self.assertLess(r.get(kf), 0.5) 

528 # sort by nonunique int32 field, try range lookups 

529 cat.sort(ki) 

530 self.assertTrue(cat.isSorted(ki)) 

531 s = cat.equal_range(3, ki) 

532 self.assertTrue(cat[s].isSorted(kf)) # test for stable sort 

533 for r in cat[s]: 

534 self.assertEqual(r.get(ki), 3) 

535 self.assertEqual(s.start, cat.lower_bound(3, ki)) 

536 self.assertEqual(s.stop, cat.upper_bound(3, ki)) 

537 

538 def testRename(self): 

539 """Test field-renaming functionality in Field, SchemaMapper. 

540 """ 

541 field1i = lsst.afw.table.Field[np.int32]("i1", "doc for i", "m") 

542 field2i = field1i.copyRenamed("i2") 

543 self.assertEqual(field1i.getName(), "i1") 

544 self.assertEqual(field2i.getName(), "i2") 

545 self.assertEqual(field1i.getDoc(), field2i.getDoc()) 

546 self.assertEqual(field1i.getUnits(), field2i.getUnits()) 

547 field1a = lsst.afw.table.Field["ArrayF"]("a1", "doc for a", "s", 3) 

548 field2a = field1a.copyRenamed("a2") 

549 self.assertEqual(field1a.getName(), "a1") 

550 self.assertEqual(field2a.getName(), "a2") 

551 self.assertEqual(field1a.getDoc(), field2a.getDoc()) 

552 self.assertEqual(field1a.getUnits(), field2a.getUnits()) 

553 self.assertEqual(field1a.getSize(), field2a.getSize()) 

554 schema1 = lsst.afw.table.Schema() 

555 k1i = schema1.addField(field1i) 

556 k1a = schema1.addField(field1a) 

557 mapper = lsst.afw.table.SchemaMapper(schema1) 

558 k2i = mapper.addMapping(k1i, "i2") 

559 k2a = mapper.addMapping(k1a, "a2") 

560 schema2 = mapper.getOutputSchema() 

561 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

562 self.assertEqual(schema2.find(k2i).field.getName(), "i2") 

563 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

564 self.assertEqual(schema2.find(k2a).field.getName(), "a2") 

565 self.assertEqual(schema1.find(k1i).field.getDoc(), 

566 schema2.find(k2i).field.getDoc()) 

567 self.assertEqual(schema1.find(k1a).field.getDoc(), 

568 schema2.find(k2a).field.getDoc()) 

569 self.assertEqual(schema1.find(k1i).field.getUnits(), 

570 schema2.find(k2i).field.getUnits()) 

571 self.assertEqual(schema1.find(k1a).field.getUnits(), 

572 schema2.find(k2a).field.getUnits()) 

573 self.assertEqual(schema1.find(k1a).field.getSize(), 

574 schema2.find(k2a).field.getSize()) 

575 k3i = mapper.addMapping(k1i, "i3") 

576 k3a = mapper.addMapping(k1a, "a3") 

577 schema3 = mapper.getOutputSchema() 

578 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

579 self.assertEqual(schema3.find(k3i).field.getName(), "i3") 

580 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

581 self.assertEqual(schema3.find(k3a).field.getName(), "a3") 

582 self.assertEqual(schema1.find(k1i).field.getDoc(), 

583 schema3.find(k3i).field.getDoc()) 

584 self.assertEqual(schema1.find(k1a).field.getDoc(), 

585 schema3.find(k3a).field.getDoc()) 

586 self.assertEqual(schema1.find(k1i).field.getUnits(), 

587 schema3.find(k3i).field.getUnits()) 

588 self.assertEqual(schema1.find(k1a).field.getUnits(), 

589 schema3.find(k3a).field.getUnits()) 

590 self.assertEqual(schema1.find(k1a).field.getSize(), 

591 schema3.find(k3a).field.getSize()) 

592 

593 def testTicket3066(self): 

594 """Test the doReplace option on Schema.addField. 

595 """ 

596 schema = lsst.afw.table.Schema() 

597 k1a = schema.addField("f1", doc="f1a", type="I") 

598 k2a = schema.addField("f2", doc="f2a", type="Flag") 

599 k3a = schema.addField("f3", doc="f3a", type="ArrayF", size=4) 

600 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

601 schema.addField("f1", doc="f1b", type="I") 

602 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

603 schema.addField("f2", doc="f2b", type="Flag") 

604 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

605 schema.addField("f1", doc="f1b", type="F") 

606 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

607 schema.addField("f2", doc="f2b", type="F") 

608 with self.assertRaises(lsst.pex.exceptions.TypeError): 

609 schema.addField("f1", doc="f1b", type="F", doReplace=True) 

610 with self.assertRaises(lsst.pex.exceptions.TypeError): 

611 schema.addField("f2", doc="f2b", type="F", doReplace=True) 

612 with self.assertRaises(lsst.pex.exceptions.TypeError): 

613 schema.addField("f3", doc="f3b", type="ArrayF", 

614 size=3, doReplace=True) 

615 k1b = schema.addField("f1", doc="f1b", type="I", doReplace=True) 

616 self.assertEqual(k1a, k1b) 

617 self.assertEqual(schema.find(k1a).field.getDoc(), "f1b") 

618 k2b = schema.addField("f2", doc="f2b", type="Flag", doReplace=True) 

619 self.assertEqual(k2a, k2b) 

620 self.assertEqual(schema.find(k2a).field.getDoc(), "f2b") 

621 k3b = schema.addField( 

622 "f3", doc="f3b", type="ArrayF", size=4, doReplace=True) 

623 self.assertEqual(k3a, k3b) 

624 self.assertEqual(schema.find(k3a).field.getDoc(), "f3b") 

625 

626 def testDM352(self): 

627 filename = os.path.join(os.path.split(__file__)[0], 

628 "data", "great3.fits") 

629 cat = lsst.afw.table.BaseCatalog.readFits(filename) 

630 self.assertEqual(len(cat), 1) 

631 

632 def testDM1710(self): 

633 # Extending without specifying a mapper or a deep argument should not 

634 # raise. 

635 schema = lsst.afw.table.Schema() 

636 cat1 = lsst.afw.table.BaseCatalog(schema) 

637 cat2 = lsst.afw.table.BaseCatalog(schema) 

638 cat1.extend(cat2) 

639 

640 def testVariableLengthArrays(self): 

641 schema = lsst.afw.table.Schema() 

642 kArrayB = schema.addField("fArrayB", doc="uint8", type="ArrayB", size=0) 

643 kArrayU = schema.addField("fArrayU", doc="uint16", type="ArrayU", size=0) 

644 kArrayI = schema.addField("fArrayI", doc="int32", type="ArrayI", size=0) 

645 kArrayF = schema.addField("fArrayF", doc="single-precision", type="ArrayF") 

646 kArrayD = schema.addField("fArrayD", doc="double-precision", type="ArrayD") 

647 kString = schema.addField("fString", doc="string", type="String", size=0) 

648 cat1 = lsst.afw.table.BaseCatalog(schema) 

649 record1 = cat1.addNew() 

650 self.assertEqual(list(record1.get(kArrayB)), []) 

651 self.assertEqual(list(record1.get(kArrayU)), []) 

652 self.assertEqual(list(record1.get(kArrayI)), []) 

653 self.assertEqual(list(record1.get(kArrayF)), []) 

654 self.assertEqual(list(record1.get(kArrayD)), []) 

655 self.assertEqual(record1.get(kString), "") 

656 dataB = np.random.randint(low=3, high=6, size=4).astype(np.uint8) 

657 dataU = np.random.randint(low=3, high=6, size=4).astype(np.uint16) 

658 dataI = np.random.randint(low=3, high=6, size=4).astype(np.int32) 

659 dataF = np.random.randn(5).astype(np.float32) 

660 dataD = np.random.randn(6).astype(np.float64) 

661 dataString = "the\nquick\tbrown\rfox jumps over the lazy dog" 

662 # Test get/set 

663 record1.set(kArrayB, dataB) 

664 record1.set(kArrayU, dataU) 

665 record1.set(kArrayI, dataI) 

666 record1.set(kArrayF, dataF) 

667 record1.set(kArrayD, dataD) 

668 record1.set(kString, dataString) 

669 self.assertFloatsEqual(record1.get(kArrayB), dataB) 

670 self.assertFloatsEqual(record1.get(kArrayU), dataU) 

671 self.assertFloatsEqual(record1.get(kArrayI), dataI) 

672 self.assertFloatsEqual(record1.get(kArrayF), dataF) 

673 self.assertFloatsEqual(record1.get(kArrayD), dataD) 

674 self.assertEqual(record1.get(kString), dataString) 

675 # Test __getitem__ and view semantics 

676 record1[kArrayF][2] = 3.5 

677 self.assertEqual(dataF[2], 3.5) 

678 # Check that we throw when we try to index a variable-length array Key 

679 with self.assertRaises(lsst.pex.exceptions.LogicError): 

680 kArrayI[0] 

681 with self.assertRaises(lsst.pex.exceptions.LogicError): 

682 kArrayI[0:1] 

683 # Test copying records, both with and without SchemaMapper 

684 record2 = cat1.addNew() 

685 record2.assign(record1) 

686 self.assertFloatsEqual(record2.get(kArrayB), dataB) 

687 self.assertFloatsEqual(record2.get(kArrayU), dataU) 

688 self.assertFloatsEqual(record2.get(kArrayI), dataI) 

689 self.assertFloatsEqual(record2.get(kArrayF), dataF) 

690 self.assertFloatsEqual(record2.get(kArrayD), dataD) 

691 self.assertEqual(record2.get(kString), dataString) 

692 record1[kArrayF][2] = 4.5 

693 # copy in assign() should be deep 

694 self.assertEqual(record2[kArrayF][2], 3.5) 

695 mapper = lsst.afw.table.SchemaMapper(schema) 

696 kb2 = mapper.addMapping(kArrayF) 

697 cat2 = lsst.afw.table.BaseCatalog(mapper.getOutputSchema()) 

698 record3 = cat2.addNew() 

699 record3.assign(record1, mapper) 

700 self.assertFloatsEqual(record3.get(kb2), dataF) 

701 # Test that we throw if we try to get a column view of a 

702 # variable-length arry 

703 with self.assertRaises(lsst.pex.exceptions.LogicError): 

704 cat1.get(kArrayI) 

705 # Test that we can round-trip variable-length arrays through FITS 

706 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

707 cat1.writeFits(filename) 

708 cat3 = lsst.afw.table.BaseCatalog.readFits(filename) 

709 self.assertEqual(schema.compare(cat3.schema, lsst.afw.table.Schema.IDENTICAL), 

710 lsst.afw.table.Schema.IDENTICAL) 

711 record4 = cat3[0] 

712 np.testing.assert_array_equal(record4.get(kArrayB), dataB) 

713 np.testing.assert_array_equal(record4.get(kArrayU), dataU) 

714 np.testing.assert_array_equal(record4.get(kArrayI), dataI) 

715 np.testing.assert_array_equal(record4.get(kArrayF), dataF) 

716 np.testing.assert_array_equal(record4.get(kArrayD), dataD) 

717 self.assertEqual(record4.get(kString), dataString) 

718 

719 def testCompoundFieldFitsConversion(self): 

720 """Test that we convert compound fields saved with an older version of the pipeline 

721 into the set of multiple fields used by their replacement FunctorKeys. 

722 """ 

723 geomValues = { 

724 "point_i_x": 4, "point_i_y": 5, 

725 "point_d_x": 3.5, "point_d_y": 2.0, 

726 "moments_xx": 5.0, "moments_yy": 6.5, "moments_xy": 2.25, 

727 "coord_ra": 1.0*lsst.geom.radians, "coord_dec": 0.5*lsst.geom.radians, 

728 } 

729 covValues = { 

730 "cov_z": np.array([[4.00, 1.25, 1.50, 0.75], 

731 [1.25, 2.25, 0.50, 0.25], 

732 [1.50, 0.50, 6.25, 1.75], 

733 [0.75, 0.25, 1.75, 9.00]], dtype=np.float32), 

734 "cov_p": np.array([[5.50, -2.0], 

735 [-2.0, 3.25]], dtype=np.float32), 

736 "cov_m": np.array([[3.75, -0.5, 1.25], 

737 [-0.5, 4.50, 0.75], 

738 [1.25, 0.75, 6.25]], dtype=np.float32), 

739 } 

740 filename = os.path.join(os.path.split(__file__)[0], 

741 "data", "CompoundFieldConversion.fits") 

742 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

743 record2 = cat2[0] 

744 for k, v in geomValues.items(): 

745 self.assertEqual(record2.get(k), v, msg=k) 

746 covZKey = lsst.afw.table.CovarianceMatrixXfKey( 

747 cat2.schema["cov_z"], ["0", "1", "2", "3"]) 

748 covPKey = lsst.afw.table.CovarianceMatrix2fKey( 

749 cat2.schema["cov_p"], ["x", "y"]) 

750 covMKey = lsst.afw.table.CovarianceMatrix3fKey( 

751 cat2.schema["cov_m"], ["xx", "yy", "xy"]) 

752 self.assertFloatsAlmostEqual(record2.get(covZKey), 

753 covValues["cov_z"], rtol=1E-6) 

754 self.assertFloatsAlmostEqual(record2.get(covPKey), 

755 covValues["cov_p"], rtol=1E-6) 

756 self.assertFloatsAlmostEqual(record2.get(covMKey), 

757 covValues["cov_m"], rtol=1E-6) 

758 

759 def testFitsReadVersion1Compatibility(self): 

760 """Test that v1 SimpleCatalogs read from FITS get correct aliases. 

761 """ 

762 filename = os.path.join(testPath, "data", "ps1-refcat-v1.fits") 

763 catalog = lsst.afw.table.SimpleCatalog.readFits(filename) 

764 self.assertIn('g_flux', catalog.schema) 

765 self.assertNotIn('g_instFlux', catalog.schema) 

766 

767 self.assertIn('g_fluxErr', catalog.schema) 

768 self.assertNotIn('g_instFluxErr', catalog.schema) 

769 

770 def testDelete(self): 

771 schema = lsst.afw.table.Schema() 

772 key = schema.addField("a", type=np.float64, doc="doc for 'a'") 

773 catalog = lsst.afw.table.BaseCatalog(schema) 

774 for i in range(10): 

775 catalog.addNew().set(key, i) 

776 del catalog[4] 

777 self.assertEqual(len(catalog), 9) 

778 self.assertEqual([r.get(key) for r in catalog], 

779 [0, 1, 2, 3, 5, 6, 7, 8, 9]) 

780 del catalog[4:7] 

781 self.assertEqual(len(catalog), 6) 

782 self.assertEqual([r.get(key) for r in catalog], 

783 [0, 1, 2, 3, 8, 9]) 

784 with self.assertRaises(IndexError): 

785 del catalog[1:3:-1] 

786 with self.assertRaises(IndexError): 

787 del catalog[:4:2] 

788 with self.assertRaises(IndexError): 

789 del catalog[50] 

790 

791 def testSetFlagColumn(self): 

792 schema = lsst.afw.table.Schema() 

793 key = schema.addField("a", type="Flag", doc="doc for 'a'") 

794 catalog = lsst.afw.table.BaseCatalog(schema) 

795 catalog.resize(5) 

796 # Set scalar with key. 

797 catalog[key] = True 

798 self.assertEqual(list(catalog[key]), [True] * 5) 

799 # Set scalar with name. 

800 catalog["a"] = False 

801 self.assertEqual(list(catalog[key]), [False] * 5) 

802 # Set array with key. 

803 v1 = np.array([True, False, True, False, True], dtype=bool) 

804 catalog[key] = v1 

805 self.assertEqual(list(catalog[key]), list(v1)) 

806 # Set array with name. 

807 v2 = np.array([False, True, False, True, False], dtype=bool) 

808 catalog["a"] = v2 

809 self.assertEqual(list(catalog[key]), list(v2)) 

810 

811 def testAngleColumnArrayAccess(self): 

812 """Test column-array access to Angle columns on both contiguous and 

813 non-contiguous arrays. 

814 """ 

815 schema = lsst.afw.table.Schema() 

816 key = schema.addField("a", type="Angle", doc="doc for a") 

817 catalog = lsst.afw.table.BaseCatalog(schema) 

818 catalog.resize(2) 

819 self.assertTrue(catalog.isContiguous()) 

820 catalog[key] = np.array([3.0, 4.0]) 

821 self.assertFloatsEqual(catalog[key], np.array([3.0, 4.0])) 

822 record = catalog.addNew() 

823 self.assertFalse(catalog.isContiguous()) 

824 record[key] = 5.0 * lsst.geom.radians 

825 self.assertFloatsEqual(catalog[key], np.array([3.0, 4.0, 5.0])) 

826 self.assertFalse(catalog[key].flags.writeable) 

827 

828 def testArrayColumnArrayAccess(self): 

829 """Test column-array access to Array columns on both contiguous and 

830 non-contiguous arrays. 

831 """ 

832 schema = lsst.afw.table.Schema() 

833 key = schema.addField("a", type="ArrayD", doc="doc for a", size=3) 

834 catalog = lsst.afw.table.BaseCatalog(schema) 

835 catalog.resize(2) 

836 self.assertTrue(catalog.isContiguous()) 

837 catalog[key] = np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]]) 

838 self.assertFloatsEqual(catalog[key], np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0]])) 

839 record = catalog.addNew() 

840 self.assertFalse(catalog.isContiguous()) 

841 record[key] = np.array([9.0, 10.0, 11.0]) 

842 self.assertFloatsEqual(catalog[key], np.array([[3.0, 4.0, 5.0], [6.0, 7.0, 8.0], [9.0, 10.0, 11.0]])) 

843 self.assertFalse(catalog[key].flags.writeable) 

844 

845 

846class MemoryTester(lsst.utils.tests.MemoryTestCase): 

847 pass 

848 

849 

850def setup_module(module): 

851 lsst.utils.tests.init() 

852 

853 

854if __name__ == "__main__": 854 ↛ 855line 854 didn't jump to line 855, because the condition on line 854 was never true

855 lsst.utils.tests.init() 

856 unittest.main()