Coverage for tests/test_simpleTable.py: 6%

642 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-06 01:56 -0800

1# This file is part of afw. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Tests for table.SimpleTable 

24 

25Run with: 

26 python test_simpleTable.py 

27or 

28 pytest test_simpleTable.py 

29""" 

30import os.path 

31import unittest 

32 

33import numpy as np 

34 

35import lsst.utils.tests 

36import lsst.pex.exceptions 

37import lsst.geom 

38import lsst.daf.base 

39import lsst.afw.table 

40import lsst.afw.fits 

41 

42# Testing files live under this, in `data/`. 

43testPath = os.path.abspath(os.path.dirname(__file__)) 

44 

45 

46def makeArray(size, dtype): 

47 return np.array(np.random.randn(size), dtype=dtype) 

48 

49 

50def makeCov(size, dtype): 

51 m = np.array(np.random.randn(size, size), dtype=dtype) 

52 # not quite symmetric for single-precision on some platforms 

53 r = np.dot(m, m.transpose()) 

54 for i in range(r.shape[0]): 

55 for j in range(i): 

56 r[i, j] = r[j, i] 

57 return r 

58 

59 

60class SimpleTableTestCase(lsst.utils.tests.TestCase): 

61 

62 def setUp(self): 

63 np.random.seed(1) 

64 

65 def checkScalarAccessors(self, record, key, name, value1, value2): 

66 fastSetter = getattr(record, "set" + key.getTypeString()) 

67 fastGetter = getattr(record, "get") 

68 record[key] = value1 

69 self.assertEqual(record[key], value1) 

70 self.assertEqual(record.get(key), value1) 

71 self.assertEqual(record[name], value1) 

72 self.assertEqual(record.get(name), value1) 

73 self.assertEqual(fastGetter(key), value1) 

74 record.set(key, value2) 

75 self.assertEqual(record[key], value2) 

76 self.assertEqual(record.get(key), value2) 

77 self.assertEqual(record[name], value2) 

78 self.assertEqual(record.get(name), value2) 

79 self.assertEqual(fastGetter(key), value2) 

80 record[name] = value1 

81 self.assertEqual(record[key], value1) 

82 self.assertEqual(record.get(key), value1) 

83 self.assertEqual(record[name], value1) 

84 self.assertEqual(record.get(name), value1) 

85 self.assertEqual(fastGetter(key), value1) 

86 record.set(name, value2) 

87 self.assertEqual(record[key], value2) 

88 self.assertEqual(record.get(key), value2) 

89 self.assertEqual(record[name], value2) 

90 self.assertEqual(record.get(name), value2) 

91 self.assertEqual(fastGetter(key), value2) 

92 fastSetter(key, value1) 

93 self.assertEqual(record[key], value1) 

94 self.assertEqual(record.get(key), value1) 

95 self.assertEqual(record[name], value1) 

96 self.assertEqual(record.get(name), value1) 

97 self.assertEqual(fastGetter(key), value1) 

98 self.assertIsNone(key.subfields) 

99 

100 def checkArrayAccessors(self, record, key, name, value): 

101 fastSetter = getattr(record, "set" + key.getTypeString()) 

102 fastGetter = getattr(record, "get") 

103 record.set(key, value) 

104 self.assertFloatsEqual(record.get(key), value) 

105 record.set(name, value) 

106 self.assertFloatsEqual(record.get(name), value) 

107 fastSetter(key, value) 

108 self.assertFloatsEqual(fastGetter(key), value) 

109 

110 def testRecordAccess(self): 

111 schema = lsst.afw.table.Schema() 

112 kB = schema.addField("fB", type="B") 

113 kU = schema.addField("fU", type="U") 

114 kI = schema.addField("fI", type="I") 

115 kL = schema.addField("fL", type="L") 

116 kF = schema.addField("fF", type="F") 

117 kD = schema.addField("fD", type="D") 

118 kAngle = schema.addField("fAngle", type="Angle") 

119 kString = schema.addField("fString", type="String", size=4) 

120 kArrayB = schema.addField("fArrayB", type="ArrayB", size=6) 

121 kArrayU = schema.addField("fArrayU", type="ArrayU", size=2) 

122 kArrayI = schema.addField("fArrayI", type="ArrayI", size=3) 

123 kArrayF = schema.addField("fArrayF", type="ArrayF", size=4) 

124 kArrayD = schema.addField("fArrayD", type="ArrayD", size=5) 

125 table = lsst.afw.table.BaseTable.make(schema) 

126 record = table.makeRecord() 

127 self.assertEqual(record[kB], 0) 

128 self.assertEqual(record[kU], 0) 

129 self.assertEqual(record[kI], 0) 

130 self.assertEqual(record[kL], 0) 

131 self.assertTrue(np.isnan(record[kF])) 

132 self.assertTrue(np.isnan(record[kD])) 

133 self.checkScalarAccessors(record, kB, "fB", 4, 5) 

134 self.checkScalarAccessors(record, kU, "fU", 5, 6) 

135 self.checkScalarAccessors(record, kI, "fI", 2, 3) 

136 self.checkScalarAccessors(record, kL, "fL", 2, 3) 

137 self.checkScalarAccessors(record, kF, "fF", 2.5, 3.5) 

138 self.checkScalarAccessors(record, kD, "fD", 2.5, 3.5) 

139 self.checkScalarAccessors(record, kAngle, "fAngle", 

140 5.1*lsst.geom.degrees, -4.1*lsst.geom.degrees) 

141 self.checkScalarAccessors(record, kString, "fString", "ab", "abcd") 

142 self.checkArrayAccessors(record, kArrayB, "fArrayB", 

143 makeArray(kArrayB.getSize(), dtype=np.uint8)) 

144 self.checkArrayAccessors(record, kArrayU, "fArrayU", 

145 makeArray(kArrayU.getSize(), dtype=np.uint16)) 

146 self.checkArrayAccessors(record, kArrayI, "fArrayI", 

147 makeArray(kArrayI.getSize(), dtype=np.int32)) 

148 self.checkArrayAccessors(record, kArrayF, "fArrayF", 

149 makeArray(kArrayF.getSize(), dtype=np.float32)) 

150 self.checkArrayAccessors(record, kArrayD, "fArrayD", 

151 makeArray(kArrayD.getSize(), dtype=np.float64)) 

152 for k in (kArrayF, kArrayD): 

153 self.assertEqual(k.subfields, tuple(range(k.getSize()))) 

154 sub1 = kArrayD.slice(1, 3) 

155 sub2 = kArrayD[0:2] 

156 self.assertFloatsAlmostEqual(record.get(sub1), 

157 record.get(kArrayD)[1:3], rtol=0, atol=0) 

158 self.assertFloatsAlmostEqual(record.get(sub2), 

159 record.get(kArrayD)[0:2], rtol=0, atol=0) 

160 self.assertEqual(sub1[0], sub2[1]) 

161 self.assertIsNone(kAngle.subfields) 

162 k0a = lsst.afw.table.Key["D"]() 

163 k0b = lsst.afw.table.Key["Flag"]() 

164 with self.assertRaises(lsst.pex.exceptions.LogicError): 

165 record.get(k0a) 

166 with self.assertRaises(lsst.pex.exceptions.LogicError): 

167 record.get(k0b) 

168 

169 def _testBaseFits(self, target): 

170 schema = lsst.afw.table.Schema() 

171 k = schema.addField("f", type="D") 

172 cat1 = lsst.afw.table.BaseCatalog(schema) 

173 for i in range(50): 

174 record = cat1.addNew() 

175 record.set(k, np.random.randn()) 

176 cat1.writeFits(target) 

177 cat2 = lsst.afw.table.BaseCatalog.readFits(target) 

178 self.assertEqual(len(cat1), len(cat2)) 

179 for r1, r2 in zip(cat1, cat2): 

180 self.assertEqual(r1.get(k), r2.get(k)) 

181 

182 def testBaseFits(self): 

183 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 

184 self._testBaseFits(tmpFile) 

185 with self.assertRaises(Exception): 

186 lsst.afw.table.BaseCatalog.readFits("nonexistentfile.fits") 

187 

188 def testMemoryFits(self): 

189 mem = lsst.afw.fits.MemFileManager() 

190 self._testBaseFits(mem) 

191 

192 def testColumnView(self): 

193 schema = lsst.afw.table.Schema() 

194 kB = schema.addField("fB", type="B") 

195 kU = schema.addField("fU", type="U") 

196 kI = schema.addField("fI", type="I") 

197 kFlag1 = schema.addField("fFlag1", type="Flag") 

198 kF = schema.addField("fF", type="F") 

199 kFlag2 = schema.addField("fFlag2", type="Flag") 

200 kD = schema.addField("fD", type="D") 

201 kFlag3 = schema.addField("fFlag3", type="Flag") 

202 kArrayF = schema.addField("fArrayF", type="ArrayF", size=2) 

203 kArrayD = schema.addField("fArrayD", type="ArrayD", size=3) 

204 kAngle = schema.addField("fAngle", type="Angle") 

205 kArrayU = schema.addField("fArrayU", type="ArrayU", size=4) 

206 catalog = lsst.afw.table.BaseCatalog(schema) 

207 catalog.addNew() 

208 catalog[0].set(kB, 5) 

209 catalog[0].set(kU, 1) 

210 catalog[0].set(kI, 2) 

211 catalog[0].set(kF, 0.5) 

212 catalog[0].set(kD, 0.25) 

213 catalog[0].set(kFlag1, False) 

214 catalog[0].set(kFlag2, True) 

215 catalog[0].set(kFlag3, False) 

216 catalog[0].set(kArrayF, np.array([-0.5, -0.25], dtype=np.float32)) 

217 catalog[0].set(kArrayD, np.array([-1.5, -1.25, 3.375], dtype=np.float64)) 

218 catalog[0].set(kAngle, lsst.geom.Angle(0.25)) 

219 catalog[0].set(kArrayU, np.array([2, 3, 4, 1], dtype=np.uint16)) 

220 col1a = catalog[kI] 

221 self.assertEqual(col1a.shape, (1,)) 

222 catalog.addNew() 

223 catalog[1].set(kB, 6) 

224 catalog[1].set(kU, 4) 

225 catalog[1].set(kI, 3) 

226 catalog[1].set(kF, 2.5) 

227 catalog[1].set(kD, 0.75) 

228 catalog[1].set(kFlag1, True) 

229 catalog[1].set(kFlag2, False) 

230 catalog[1].set(kFlag3, True) 

231 catalog[1].set(kArrayF, np.array([-3.25, -0.75], dtype=np.float32)) 

232 catalog[1].set(kArrayD, np.array([-1.25, -2.75, 0.625], dtype=np.float64)) 

233 catalog[1].set(kAngle, lsst.geom.Angle(0.15)) 

234 catalog[1].set(kArrayU, np.array([5, 6, 8, 7], dtype=np.uint16)) 

235 col1b = catalog[kI] 

236 self.assertEqual(col1b.shape, (2,)) 

237 columns = catalog.getColumnView() 

238 for key in [kB, kU, kI, kF, kD, kFlag1, kFlag2, kFlag3]: 

239 array = columns[key] 

240 for i in [0, 1]: 

241 self.assertEqual(array[i], catalog[i].get(key)) 

242 for key in [kArrayF, kArrayD, kArrayU]: 

243 array = columns[key] 

244 for i in [0, 1]: 

245 self.assertFloatsEqual(array[i], catalog[i].get(key)) 

246 for key in [kAngle]: 

247 array = columns[key] 

248 for i in [0, 1]: 

249 self.assertEqual(lsst.geom.Angle(array[i]), 

250 catalog[i].get(key)) 

251 for key in [kB, kU, kI, kF, kD]: 

252 vals = columns[key].copy() 

253 vals *= 2 

254 array = columns[key] 

255 array *= 2 

256 for i in [0, 1]: 

257 self.assertEqual(catalog[i].get(key), vals[i]) 

258 self.assertEqual(array[i], vals[i]) 

259 catalog[kI] = 4 

260 f3v = np.random.randn(2) 

261 catalog["fD"] = f3v 

262 for i in [0, 1]: 

263 self.assertEqual(catalog[i].get(kI), 4) 

264 self.assertEqual(catalog[i].get(kD), f3v[i]) 

265 

266 # Accessing an invalid key should raise. 

267 for keyType in ["Angle", "ArrayB", "ArrayD", "ArrayF", "ArrayI", 

268 "ArrayU", "B", "D", "F", "Flag", "I", "L", "U"]: 

269 # Default-constructed key is invalid 

270 invalidKey = getattr(lsst.afw.table, f"Key{keyType}")() 

271 self.assertFalse(invalidKey.isValid()) 

272 with self.assertRaises(lsst.pex.exceptions.LogicError): 

273 catalog.get(invalidKey) 

274 

275 def testUnsignedFitsPersistence(self): 

276 """Test FITS round-trip of unsigned short ints, since FITS handles unsigned columns differently 

277 from signed columns. 

278 """ 

279 schema = lsst.afw.table.Schema() 

280 k1 = schema.addField("f1", type=np.uint16, doc="scalar uint16") 

281 k2 = schema.addField("f2", type="ArrayU", doc="array uint16", size=4) 

282 cat1 = lsst.afw.table.BaseCatalog(schema) 

283 record1 = cat1.addNew() 

284 record1.set(k1, 4) 

285 record1.set(k2, np.array([5, 6, 7, 8], dtype=np.uint16)) 

286 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

287 cat1.writeFits(filename) 

288 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

289 record2 = cat2[0] 

290 self.assertEqual(cat1.schema, cat2.schema) 

291 self.assertEqual(record1.get(k1), record2.get(k1)) 

292 self.assertFloatsEqual(record1.get(k2), record2.get(k2)) 

293 

294 def testIteration(self): 

295 schema = lsst.afw.table.Schema() 

296 k = schema.addField("a", type=np.int32) 

297 catalog = lsst.afw.table.BaseCatalog(schema) 

298 for n in range(5): 

299 record = catalog.addNew() 

300 record[k] = n 

301 for n, r in enumerate(catalog): 

302 self.assertEqual(n, r[k]) 

303 

304 def testTicket2262(self): 

305 """Test that we can construct an array field in Python. 

306 """ 

307 f1 = lsst.afw.table.Field["ArrayF"]("name", "doc", "barn", size=5) 

308 f2 = lsst.afw.table.Field["ArrayD"]("name", "doc", size=5) 

309 self.assertEqual(f1.getSize(), 5) 

310 self.assertEqual(f2.getSize(), 5) 

311 

312 def testExtract(self): 

313 schema = lsst.afw.table.Schema() 

314 schema.addField("a_b_c1", type=np.float64) 

315 schema.addField("a_b_c2", type="Flag") 

316 schema.addField("a_d1", type=np.int32) 

317 schema.addField("a_d2", type=np.float32) 

318 pointKey = lsst.afw.table.Point2IKey.addFields( 

319 schema, "q_e1", "doc for point field", "pixel") 

320 schema.addField("q_e2_xxErr", type=np.float32) 

321 schema.addField("q_e2_yyErr", type=np.float32) 

322 schema.addField("q_e2_xyErr", type=np.float32) 

323 schema.addField("q_e2_xx_yy_Cov", type=np.float32) 

324 schema.addField("q_e2_xx_xy_Cov", type=np.float32) 

325 schema.addField("q_e2_yy_xy_Cov", type=np.float32) 

326 covKey = lsst.afw.table.CovarianceMatrix3fKey( 

327 schema["q_e2"], ["xx", "yy", "xy"]) 

328 self.assertEqual( 

329 list(schema.extract("a_b_*", ordered=True).keys()), ["a_b_c1", "a_b_c2"]) 

330 self.assertEqual( 

331 list(schema.extract("*1", ordered=True).keys()), ["a_b_c1", "a_d1"]) 

332 self.assertEqual(list(schema.extract("a_b_*", "*2", ordered=True).keys()), 

333 ["a_b_c1", "a_b_c2", "a_d2"]) 

334 self.assertEqual(list(schema.extract(regex=r"a_(.+)1", sub=r"\1f", 

335 ordered=True).keys()), ["b_cf", "df"]) 

336 catalog = lsst.afw.table.BaseCatalog(schema) 

337 for i in range(5): 

338 record = catalog.addNew() 

339 record.set("a_b_c1", np.random.randn()) 

340 record.set("a_b_c2", True) 

341 record.set("a_d1", np.random.randint(100)) 

342 record.set("a_d2", np.random.randn()) 

343 record.set(pointKey, 

344 lsst.geom.Point2I(np.random.randint(10), np.random.randint(10))) 

345 record.set(covKey, np.random.randn(3, 3).astype(np.float32)) 

346 d = record.extract("*") 

347 self.assertEqual(set(d.keys()), set(schema.getNames())) 

348 self.assertEqual(d["a_b_c1"], record.get("a_b_c1")) 

349 self.assertEqual(d["a_b_c2"], record.get("a_b_c2")) 

350 self.assertEqual(d["a_d1"], record.get("a_d1")) 

351 self.assertEqual(d["a_d2"], record.get("a_d2")) 

352 self.assertEqual(d["q_e1_x"], record.get(pointKey.getX())) 

353 self.assertEqual(d["q_e1_y"], record.get(pointKey.getY())) 

354 allIdx = slice(None) 

355 sliceIdx = slice(0, 4, 2) 

356 boolIdx = np.array([True, False, False, True, True]) 

357 for kwds, idx in [ 

358 ({}, allIdx), 

359 ({"copy": True}, allIdx), 

360 ({"where": boolIdx}, boolIdx), 

361 ({"where": sliceIdx}, sliceIdx), 

362 ({"where": boolIdx, "copy": True}, boolIdx), 

363 ({"where": sliceIdx, "copy": True}, sliceIdx), 

364 ]: 

365 d = catalog.extract("*", **kwds) 

366 np.testing.assert_array_equal( 

367 d["a_b_c1"], catalog.get("a_b_c1")[idx]) 

368 np.testing.assert_array_equal( 

369 d["a_b_c2"], catalog.get("a_b_c2")[idx]) 

370 np.testing.assert_array_equal( 

371 d["a_d1"], catalog.get("a_d1")[idx]) 

372 np.testing.assert_array_equal( 

373 d["a_d2"], catalog.get("a_d2")[idx]) 

374 np.testing.assert_array_equal( 

375 d["q_e1_x"], catalog.get("q_e1_x")[idx]) 

376 np.testing.assert_array_equal( 

377 d["q_e1_y"], catalog.get("q_e1_y")[idx]) 

378 if "copy" in kwds or idx is boolIdx: 

379 for col in d.values(): 

380 self.assertTrue(col.flags.c_contiguous) 

381 # Test that aliases are included in extract() 

382 schema.getAliasMap().set("b_f", "a_b") 

383 d = schema.extract("b_f*") 

384 self.assertEqual(sorted(d.keys()), ["b_f_c1", "b_f_c2"]) 

385 

386 def testExtend(self): 

387 schema1 = lsst.afw.table.SourceTable.makeMinimalSchema() 

388 k1 = schema1.addField("f1", type=np.int32) 

389 k2 = schema1.addField("f2", type=np.float64) 

390 cat1 = lsst.afw.table.BaseCatalog(schema1) 

391 for i in range(1000): 

392 record = cat1.addNew() 

393 record.setI(k1, i) 

394 record.setD(k2, np.random.randn()) 

395 self.assertFalse(cat1.isContiguous()) 

396 cat2 = lsst.afw.table.BaseCatalog(schema1) 

397 cat2.extend(cat1, deep=True) 

398 self.assertEqual(len(cat1), len(cat2)) 

399 self.assertTrue(cat2.isContiguous()) 

400 cat3 = lsst.afw.table.BaseCatalog(cat1.table) 

401 cat3.extend(cat1, deep=False) 

402 self.assertFalse(cat3.isContiguous()) 

403 cat4 = lsst.afw.table.BaseCatalog(cat1.table) 

404 cat4.extend(list(cat1), deep=False) 

405 self.assertFalse(cat4.isContiguous()) 

406 cat4 = lsst.afw.table.BaseCatalog(schema1) 

407 cat4.extend(list(cat1), deep=True) 

408 self.assertFalse(cat4.isContiguous()) 

409 mapper = lsst.afw.table.SchemaMapper(schema1) 

410 mapper.addMinimalSchema(lsst.afw.table.SourceTable.makeMinimalSchema()) 

411 mapper.addMapping(k2) 

412 schema2 = mapper.getOutputSchema() 

413 self.assertTrue(mapper.getOutputSchema().contains( 

414 lsst.afw.table.SourceTable.makeMinimalSchema())) 

415 cat5 = lsst.afw.table.BaseCatalog(schema2) 

416 cat5.extend(cat1, mapper=mapper) 

417 self.assertTrue(cat5.isContiguous()) 

418 cat6 = lsst.afw.table.SourceCatalog(schema2) 

419 cat6.extend(list(cat1), mapper=mapper) 

420 self.assertFalse(cat6.isContiguous()) 

421 cat7 = lsst.afw.table.SourceCatalog(schema2) 

422 cat7.reserve(len(cat1)*3) 

423 cat7.extend(list(cat1), mapper=mapper) 

424 cat7.extend(cat1, mapper) 

425 cat7.extend(list(cat1), mapper) 

426 self.assertTrue(cat7.isContiguous()) 

427 cat8 = lsst.afw.table.BaseCatalog(schema2) 

428 cat8.extend(list(cat7), True) 

429 cat8.extend(list(cat7), deep=True) 

430 

431 def testTicket2308(self): 

432 inputSchema = lsst.afw.table.SourceTable.makeMinimalSchema() 

433 mapper1 = lsst.afw.table.SchemaMapper(inputSchema) 

434 mapper1.addMinimalSchema( 

435 lsst.afw.table.SourceTable.makeMinimalSchema(), True) 

436 mapper2 = lsst.afw.table.SchemaMapper(inputSchema) 

437 mapper2.addMinimalSchema( 

438 lsst.afw.table.SourceTable.makeMinimalSchema(), False) 

439 inputTable = lsst.afw.table.SourceTable.make(inputSchema) 

440 inputRecord = inputTable.makeRecord() 

441 inputRecord.set("id", 42) 

442 outputTable1 = lsst.afw.table.SourceTable.make( 

443 mapper1.getOutputSchema()) 

444 outputTable2 = lsst.afw.table.SourceTable.make( 

445 mapper2.getOutputSchema()) 

446 outputRecord1 = outputTable1.makeRecord() 

447 outputRecord2 = outputTable2.makeRecord() 

448 self.assertEqual(outputRecord1.getId(), outputRecord2.getId()) 

449 self.assertNotEqual(outputRecord1.getId(), inputRecord.getId()) 

450 outputRecord1.assign(inputRecord, mapper1) 

451 self.assertEqual(outputRecord1.getId(), inputRecord.getId()) 

452 outputRecord2.assign(inputRecord, mapper2) 

453 self.assertNotEqual(outputRecord2.getId(), inputRecord.getId()) 

454 

455 def testTicket2393(self): 

456 schema = lsst.afw.table.Schema() 

457 k = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

458 item = schema.find("i") 

459 self.assertEqual(k, item.key) 

460 

461 def testTicket2850(self): 

462 schema = lsst.afw.table.Schema() 

463 table = lsst.afw.table.BaseTable.make(schema) 

464 self.assertEqual(table.getBufferSize(), 0) 

465 

466 def testTicket2894(self): 

467 """Test boolean-array indexing of catalogs. 

468 """ 

469 schema = lsst.afw.table.Schema() 

470 key = schema.addField(lsst.afw.table.Field[np.int32]("i", "doc for i")) 

471 cat1 = lsst.afw.table.BaseCatalog(schema) 

472 cat1.addNew().set(key, 1) 

473 cat1.addNew().set(key, 2) 

474 cat1.addNew().set(key, 3) 

475 cat2 = cat1[np.array([True, False, False], dtype=bool)] 

476 self.assertFloatsEqual(cat2[key], np.array([1], dtype=np.int32)) 

477 # records compare using pointer equality 

478 self.assertEqual(cat2[0], cat1[0]) 

479 cat3 = cat1[np.array([True, True, False], dtype=bool)] 

480 self.assertFloatsEqual(cat3[key], np.array([1, 2], dtype=np.int32)) 

481 cat4 = cat1[np.array([True, False, True], dtype=bool)] 

482 self.assertFloatsEqual(cat4.copy(deep=True)[ 

483 key], np.array([1, 3], dtype=np.int32)) 

484 

485 def testTicket2938(self): 

486 """Test heterogenous catalogs that have records from multiple tables. 

487 """ 

488 schema = lsst.afw.table.Schema() 

489 schema.addField("i", type=np.int32, doc="doc for i") 

490 cat = lsst.afw.table.BaseCatalog(schema) 

491 cat.addNew() 

492 t1 = lsst.afw.table.BaseTable.make(schema) 

493 cat.append(t1.makeRecord()) 

494 self.assertEqual(cat[-1].getTable(), t1) 

495 with self.assertRaises(lsst.pex.exceptions.RuntimeError): 

496 cat.getColumnView() 

497 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

498 cat.writeFits(filename) # shouldn't throw 

499 schema.addField("d", type=np.float64, doc="doc for d") 

500 t2 = lsst.afw.table.BaseTable.make(schema) 

501 cat.append(t2.makeRecord()) 

502 with self.assertRaises(lsst.pex.exceptions.LogicError): 

503 cat.writeFits(filename) 

504 

505 def testTicket3056(self): 

506 """Test sorting and sort-based searches of Catalogs. 

507 """ 

508 schema = lsst.afw.table.SimpleTable.makeMinimalSchema() 

509 ki = schema.addField("i", type=np.int32, doc="doc for i") 

510 kl = schema.addField("l", type=np.int64, doc="doc for l") 

511 kf = schema.addField("f", type=np.float64, doc="doc for f") 

512 cat = lsst.afw.table.SimpleCatalog(schema) 

513 for j in range(50, 0, -1): 

514 record = cat.addNew() 

515 record.set(ki, j//10) 

516 record.set(kl, j) 

517 record.set(kf, np.random.randn()) 

518 self.assertFalse(cat.isSorted(ki)) 

519 self.assertFalse(cat.isSorted(kl)) 

520 # sort by unique int64 field, try unique lookups 

521 cat.sort(kl) 

522 self.assertTrue(cat.isSorted(kl)) 

523 r10 = cat.find(10, kl) 

524 self.assertEqual(r10.get(kl), 10) 

525 # sort by probably-unique float field, try unique and range lookups 

526 cat.sort(kf) 

527 self.assertTrue(cat.isSorted(kf)) 

528 r10 = cat.find(10, kf) 

529 # latter case virtually impossible 

530 self.assertTrue(r10 is None or r10.get(kf) == 10.0) 

531 i0 = cat.lower_bound(-0.5, kf) 

532 i1 = cat.upper_bound(0.5, kf) 

533 for i in range(i0, i1): 

534 self.assertGreaterEqual(cat[i].get(kf), -0.5) 

535 self.assertLess(cat[i].get(kf), 0.5) 

536 for r in cat[cat.between(-0.5, 0.5, kf)]: 

537 self.assertGreaterEqual(r.get(kf), -0.5) 

538 self.assertLess(r.get(kf), 0.5) 

539 # sort by nonunique int32 field, try range lookups 

540 cat.sort(ki) 

541 self.assertTrue(cat.isSorted(ki)) 

542 s = cat.equal_range(3, ki) 

543 self.assertTrue(cat[s].isSorted(kf)) # test for stable sort 

544 for r in cat[s]: 

545 self.assertEqual(r.get(ki), 3) 

546 self.assertEqual(s.start, cat.lower_bound(3, ki)) 

547 self.assertEqual(s.stop, cat.upper_bound(3, ki)) 

548 

549 def testRename(self): 

550 """Test field-renaming functionality in Field, SchemaMapper. 

551 """ 

552 field1i = lsst.afw.table.Field[np.int32]("i1", "doc for i", "m") 

553 field2i = field1i.copyRenamed("i2") 

554 self.assertEqual(field1i.getName(), "i1") 

555 self.assertEqual(field2i.getName(), "i2") 

556 self.assertEqual(field1i.getDoc(), field2i.getDoc()) 

557 self.assertEqual(field1i.getUnits(), field2i.getUnits()) 

558 field1a = lsst.afw.table.Field["ArrayF"]("a1", "doc for a", "s", 3) 

559 field2a = field1a.copyRenamed("a2") 

560 self.assertEqual(field1a.getName(), "a1") 

561 self.assertEqual(field2a.getName(), "a2") 

562 self.assertEqual(field1a.getDoc(), field2a.getDoc()) 

563 self.assertEqual(field1a.getUnits(), field2a.getUnits()) 

564 self.assertEqual(field1a.getSize(), field2a.getSize()) 

565 schema1 = lsst.afw.table.Schema() 

566 k1i = schema1.addField(field1i) 

567 k1a = schema1.addField(field1a) 

568 mapper = lsst.afw.table.SchemaMapper(schema1) 

569 k2i = mapper.addMapping(k1i, "i2") 

570 k2a = mapper.addMapping(k1a, "a2") 

571 schema2 = mapper.getOutputSchema() 

572 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

573 self.assertEqual(schema2.find(k2i).field.getName(), "i2") 

574 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

575 self.assertEqual(schema2.find(k2a).field.getName(), "a2") 

576 self.assertEqual(schema1.find(k1i).field.getDoc(), 

577 schema2.find(k2i).field.getDoc()) 

578 self.assertEqual(schema1.find(k1a).field.getDoc(), 

579 schema2.find(k2a).field.getDoc()) 

580 self.assertEqual(schema1.find(k1i).field.getUnits(), 

581 schema2.find(k2i).field.getUnits()) 

582 self.assertEqual(schema1.find(k1a).field.getUnits(), 

583 schema2.find(k2a).field.getUnits()) 

584 self.assertEqual(schema1.find(k1a).field.getSize(), 

585 schema2.find(k2a).field.getSize()) 

586 k3i = mapper.addMapping(k1i, "i3") 

587 k3a = mapper.addMapping(k1a, "a3") 

588 schema3 = mapper.getOutputSchema() 

589 self.assertEqual(schema1.find(k1i).field.getName(), "i1") 

590 self.assertEqual(schema3.find(k3i).field.getName(), "i3") 

591 self.assertEqual(schema1.find(k1a).field.getName(), "a1") 

592 self.assertEqual(schema3.find(k3a).field.getName(), "a3") 

593 self.assertEqual(schema1.find(k1i).field.getDoc(), 

594 schema3.find(k3i).field.getDoc()) 

595 self.assertEqual(schema1.find(k1a).field.getDoc(), 

596 schema3.find(k3a).field.getDoc()) 

597 self.assertEqual(schema1.find(k1i).field.getUnits(), 

598 schema3.find(k3i).field.getUnits()) 

599 self.assertEqual(schema1.find(k1a).field.getUnits(), 

600 schema3.find(k3a).field.getUnits()) 

601 self.assertEqual(schema1.find(k1a).field.getSize(), 

602 schema3.find(k3a).field.getSize()) 

603 

604 def testTicket3066(self): 

605 """Test the doReplace option on Schema.addField. 

606 """ 

607 schema = lsst.afw.table.Schema() 

608 k1a = schema.addField("f1", doc="f1a", type="I") 

609 k2a = schema.addField("f2", doc="f2a", type="Flag") 

610 k3a = schema.addField("f3", doc="f3a", type="ArrayF", size=4) 

611 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

612 schema.addField("f1", doc="f1b", type="I") 

613 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

614 schema.addField("f2", doc="f2b", type="Flag") 

615 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

616 schema.addField("f1", doc="f1b", type="F") 

617 with self.assertRaises(lsst.pex.exceptions.InvalidParameterError): 

618 schema.addField("f2", doc="f2b", type="F") 

619 with self.assertRaises(lsst.pex.exceptions.TypeError): 

620 schema.addField("f1", doc="f1b", type="F", doReplace=True) 

621 with self.assertRaises(lsst.pex.exceptions.TypeError): 

622 schema.addField("f2", doc="f2b", type="F", doReplace=True) 

623 with self.assertRaises(lsst.pex.exceptions.TypeError): 

624 schema.addField("f3", doc="f3b", type="ArrayF", 

625 size=3, doReplace=True) 

626 k1b = schema.addField("f1", doc="f1b", type="I", doReplace=True) 

627 self.assertEqual(k1a, k1b) 

628 self.assertEqual(schema.find(k1a).field.getDoc(), "f1b") 

629 k2b = schema.addField("f2", doc="f2b", type="Flag", doReplace=True) 

630 self.assertEqual(k2a, k2b) 

631 self.assertEqual(schema.find(k2a).field.getDoc(), "f2b") 

632 k3b = schema.addField( 

633 "f3", doc="f3b", type="ArrayF", size=4, doReplace=True) 

634 self.assertEqual(k3a, k3b) 

635 self.assertEqual(schema.find(k3a).field.getDoc(), "f3b") 

636 

637 def testDM352(self): 

638 filename = os.path.join(os.path.split(__file__)[0], 

639 "data", "great3.fits") 

640 cat = lsst.afw.table.BaseCatalog.readFits(filename) 

641 self.assertEqual(len(cat), 1) 

642 

643 def testDM1710(self): 

644 # Extending without specifying a mapper or a deep argument should not 

645 # raise. 

646 schema = lsst.afw.table.Schema() 

647 cat1 = lsst.afw.table.BaseCatalog(schema) 

648 cat2 = lsst.afw.table.BaseCatalog(schema) 

649 cat1.extend(cat2) 

650 

651 def testVariableLengthArrays(self): 

652 schema = lsst.afw.table.Schema() 

653 kArrayB = schema.addField("fArrayB", doc="uint8", type="ArrayB", size=0) 

654 kArrayU = schema.addField("fArrayU", doc="uint16", type="ArrayU", size=0) 

655 kArrayI = schema.addField("fArrayI", doc="int32", type="ArrayI", size=0) 

656 kArrayF = schema.addField("fArrayF", doc="single-precision", type="ArrayF") 

657 kArrayD = schema.addField("fArrayD", doc="double-precision", type="ArrayD") 

658 kString = schema.addField("fString", doc="string", type="String", size=0) 

659 cat1 = lsst.afw.table.BaseCatalog(schema) 

660 record1 = cat1.addNew() 

661 self.assertEqual(list(record1.get(kArrayB)), []) 

662 self.assertEqual(list(record1.get(kArrayU)), []) 

663 self.assertEqual(list(record1.get(kArrayI)), []) 

664 self.assertEqual(list(record1.get(kArrayF)), []) 

665 self.assertEqual(list(record1.get(kArrayD)), []) 

666 self.assertEqual(record1.get(kString), "") 

667 dataB = np.random.randint(low=3, high=6, size=4).astype(np.uint8) 

668 dataU = np.random.randint(low=3, high=6, size=4).astype(np.uint16) 

669 dataI = np.random.randint(low=3, high=6, size=4).astype(np.int32) 

670 dataF = np.random.randn(5).astype(np.float32) 

671 dataD = np.random.randn(6).astype(np.float64) 

672 dataString = "the\nquick\tbrown\rfox jumps over the lazy dog" 

673 # Test get/set 

674 record1.set(kArrayB, dataB) 

675 record1.set(kArrayU, dataU) 

676 record1.set(kArrayI, dataI) 

677 record1.set(kArrayF, dataF) 

678 record1.set(kArrayD, dataD) 

679 record1.set(kString, dataString) 

680 self.assertFloatsEqual(record1.get(kArrayB), dataB) 

681 self.assertFloatsEqual(record1.get(kArrayU), dataU) 

682 self.assertFloatsEqual(record1.get(kArrayI), dataI) 

683 self.assertFloatsEqual(record1.get(kArrayF), dataF) 

684 self.assertFloatsEqual(record1.get(kArrayD), dataD) 

685 self.assertEqual(record1.get(kString), dataString) 

686 # Test __getitem__ and view semantics 

687 record1[kArrayF][2] = 3.5 

688 self.assertEqual(dataF[2], 3.5) 

689 # Check that we throw when we try to index a variable-length array Key 

690 with self.assertRaises(lsst.pex.exceptions.LogicError): 

691 kArrayI[0] 

692 with self.assertRaises(lsst.pex.exceptions.LogicError): 

693 kArrayI[0:1] 

694 # Test copying records, both with and without SchemaMapper 

695 record2 = cat1.addNew() 

696 record2.assign(record1) 

697 self.assertFloatsEqual(record2.get(kArrayB), dataB) 

698 self.assertFloatsEqual(record2.get(kArrayU), dataU) 

699 self.assertFloatsEqual(record2.get(kArrayI), dataI) 

700 self.assertFloatsEqual(record2.get(kArrayF), dataF) 

701 self.assertFloatsEqual(record2.get(kArrayD), dataD) 

702 self.assertEqual(record2.get(kString), dataString) 

703 record1[kArrayF][2] = 4.5 

704 # copy in assign() should be deep 

705 self.assertEqual(record2[kArrayF][2], 3.5) 

706 mapper = lsst.afw.table.SchemaMapper(schema) 

707 kb2 = mapper.addMapping(kArrayF) 

708 cat2 = lsst.afw.table.BaseCatalog(mapper.getOutputSchema()) 

709 record3 = cat2.addNew() 

710 record3.assign(record1, mapper) 

711 self.assertFloatsEqual(record3.get(kb2), dataF) 

712 # Test that we throw if we try to get a column view of a 

713 # variable-length arry 

714 with self.assertRaises(lsst.pex.exceptions.LogicError): 

715 cat1.get(kArrayI) 

716 # Test that we can round-trip variable-length arrays through FITS 

717 with lsst.utils.tests.getTempFilePath(".fits") as filename: 

718 cat1.writeFits(filename) 

719 cat3 = lsst.afw.table.BaseCatalog.readFits(filename) 

720 self.assertEqual(schema.compare(cat3.schema, lsst.afw.table.Schema.IDENTICAL), 

721 lsst.afw.table.Schema.IDENTICAL) 

722 record4 = cat3[0] 

723 np.testing.assert_array_equal(record4.get(kArrayB), dataB) 

724 np.testing.assert_array_equal(record4.get(kArrayU), dataU) 

725 np.testing.assert_array_equal(record4.get(kArrayI), dataI) 

726 np.testing.assert_array_equal(record4.get(kArrayF), dataF) 

727 np.testing.assert_array_equal(record4.get(kArrayD), dataD) 

728 self.assertEqual(record4.get(kString), dataString) 

729 

730 def testCompoundFieldFitsConversion(self): 

731 """Test that we convert compound fields saved with an older version of the pipeline 

732 into the set of multiple fields used by their replacement FunctorKeys. 

733 """ 

734 geomValues = { 

735 "point_i_x": 4, "point_i_y": 5, 

736 "point_d_x": 3.5, "point_d_y": 2.0, 

737 "moments_xx": 5.0, "moments_yy": 6.5, "moments_xy": 2.25, 

738 "coord_ra": 1.0*lsst.geom.radians, "coord_dec": 0.5*lsst.geom.radians, 

739 } 

740 covValues = { 

741 "cov_z": np.array([[4.00, 1.25, 1.50, 0.75], 

742 [1.25, 2.25, 0.50, 0.25], 

743 [1.50, 0.50, 6.25, 1.75], 

744 [0.75, 0.25, 1.75, 9.00]], dtype=np.float32), 

745 "cov_p": np.array([[5.50, -2.0], 

746 [-2.0, 3.25]], dtype=np.float32), 

747 "cov_m": np.array([[3.75, -0.5, 1.25], 

748 [-0.5, 4.50, 0.75], 

749 [1.25, 0.75, 6.25]], dtype=np.float32), 

750 } 

751 filename = os.path.join(os.path.split(__file__)[0], 

752 "data", "CompoundFieldConversion.fits") 

753 cat2 = lsst.afw.table.BaseCatalog.readFits(filename) 

754 record2 = cat2[0] 

755 for k, v in geomValues.items(): 

756 self.assertEqual(record2.get(k), v, msg=k) 

757 covZKey = lsst.afw.table.CovarianceMatrixXfKey( 

758 cat2.schema["cov_z"], ["0", "1", "2", "3"]) 

759 covPKey = lsst.afw.table.CovarianceMatrix2fKey( 

760 cat2.schema["cov_p"], ["x", "y"]) 

761 covMKey = lsst.afw.table.CovarianceMatrix3fKey( 

762 cat2.schema["cov_m"], ["xx", "yy", "xy"]) 

763 self.assertFloatsAlmostEqual(record2.get(covZKey), 

764 covValues["cov_z"], rtol=1E-6) 

765 self.assertFloatsAlmostEqual(record2.get(covPKey), 

766 covValues["cov_p"], rtol=1E-6) 

767 self.assertFloatsAlmostEqual(record2.get(covMKey), 

768 covValues["cov_m"], rtol=1E-6) 

769 

770 def testFitsReadVersion1Compatibility(self): 

771 """Test that v1 SimpleCatalogs read from FITS get correct aliases. 

772 """ 

773 filename = os.path.join(testPath, "data", "ps1-refcat-v1.fits") 

774 catalog = lsst.afw.table.SimpleCatalog.readFits(filename) 

775 self.assertIn('g_flux', catalog.schema) 

776 self.assertNotIn('g_instFlux', catalog.schema) 

777 

778 self.assertIn('g_fluxErr', catalog.schema) 

779 self.assertNotIn('g_instFluxErr', catalog.schema) 

780 

781 def testDelete(self): 

782 schema = lsst.afw.table.Schema() 

783 key = schema.addField("a", type=np.float64, doc="doc for 'a'") 

784 catalog = lsst.afw.table.BaseCatalog(schema) 

785 for i in range(10): 

786 catalog.addNew().set(key, i) 

787 del catalog[4] 

788 self.assertEqual(len(catalog), 9) 

789 self.assertEqual([r.get(key) for r in catalog], 

790 [0, 1, 2, 3, 5, 6, 7, 8, 9]) 

791 del catalog[4:7] 

792 self.assertEqual(len(catalog), 6) 

793 self.assertEqual([r.get(key) for r in catalog], 

794 [0, 1, 2, 3, 8, 9]) 

795 with self.assertRaises(IndexError): 

796 del catalog[1:3:-1] 

797 with self.assertRaises(IndexError): 

798 del catalog[:4:2] 

799 with self.assertRaises(IndexError): 

800 del catalog[50] 

801 

802 def testSetFlagColumn(self): 

803 schema = lsst.afw.table.Schema() 

804 key = schema.addField("a", type="Flag", doc="doc for 'a'") 

805 catalog = lsst.afw.table.BaseCatalog(schema) 

806 catalog.resize(5) 

807 # Set scalar with key. 

808 catalog[key] = True 

809 self.assertEqual(list(catalog[key]), [True] * 5) 

810 # Set scalar with name. 

811 catalog["a"] = False 

812 self.assertEqual(list(catalog[key]), [False] * 5) 

813 # Set array with key. 

814 v1 = np.array([True, False, True, False, True], dtype=bool) 

815 catalog[key] = v1 

816 self.assertEqual(list(catalog[key]), list(v1)) 

817 # Set array with name. 

818 v2 = np.array([False, True, False, True, False], dtype=bool) 

819 catalog["a"] = v2 

820 self.assertEqual(list(catalog[key]), list(v2)) 

821 

822 

823class MemoryTester(lsst.utils.tests.MemoryTestCase): 

824 pass 

825 

826 

827def setup_module(module): 

828 lsst.utils.tests.init() 

829 

830 

831if __name__ == "__main__": 831 ↛ 832line 831 didn't jump to line 832, because the condition on line 831 was never true

832 lsst.utils.tests.init() 

833 unittest.main()