Coverage for tests/test_cameraMapper.py: 21%

397 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-09 06:33 +0000

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import gc 

23import os 

24import shutil 

25import sqlite3 

26import tempfile 

27import unittest 

28 

29import lsst.afw.image as afwImage 

30import lsst.afw.table as afwTable 

31import lsst.daf.persistence as dafPersist 

32import lsst.geom as geom 

33import lsst.obs.base 

34import lsst.utils.tests 

35import numpy as np 

36from lsst.obs.base.test import BaseMapper 

37 

38ROOT = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41def setup_module(module): 

42 lsst.utils.tests.init() 

43 

44 

45class MinCam(lsst.obs.base.Instrument): 

46 @property 

47 def filterDefinitions(self): 

48 return lsst.obs.base.FilterDefinitionCollection( 

49 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u"), 

50 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g"), 

51 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"}), 

52 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"}), 

53 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z"), 

54 # afw_name is so special-cased that only a real example will work 

55 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2"), 

56 ) 

57 

58 @classmethod 

59 def getName(cls): 

60 return "min" 

61 

62 def getCamera(self): 

63 raise NotImplementedError() 

64 

65 def register(self, registry): 

66 raise NotImplementedError() 

67 

68 def getRawFormatter(self, dataId): 

69 raise NotImplementedError() 

70 

71 def makeDataIdTranslatorFactory(self): 

72 raise NotImplementedError() 

73 

74 

75class MinMapper1(lsst.obs.base.CameraMapper): 

76 packageName = "larry" 

77 

78 def __init__(self, **kwargs): 

79 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

80 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

81 return 

82 

83 def std_x(self, item, dataId): 

84 return float(item) 

85 

86 @classmethod 

87 def getCameraName(cls): 

88 """Return the name of the camera that this CameraMapper is for.""" 

89 return "min" 

90 

91 @classmethod 

92 def getPackageDir(cls): 

93 return "/path/to/nowhere" 

94 

95 

96class MinMapper2(lsst.obs.base.CameraMapper): 

97 packageName = "moe" 

98 _gen3instrument = MinCam 

99 

100 # CalibRoot in policy 

101 # needCalibRegistry 

102 def __init__(self, **kwargs): 

103 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml")) 

104 lsst.obs.base.CameraMapper.__init__( 

105 self, policy=policy, repositoryDir=ROOT, registry="cfhtls.sqlite3", **kwargs 

106 ) 

107 return 

108 

109 def _transformId(self, dataId): 

110 return dataId 

111 

112 def _extractDetectorName(self, dataId): 

113 return "ccd00" 

114 

115 def std_x(self, item, dataId): 

116 return float(item) 

117 

118 @classmethod 

119 def getCameraName(cls): 

120 """Return the name of the camera that this CameraMapper is for.""" 

121 return "min" 

122 

123 @classmethod 

124 def getPackageDir(cls): 

125 return "/path/to/nowhere" 

126 

127 

128# does not assign packageName 

129class MinMapper3(lsst.obs.base.CameraMapper): 

130 def __init__(self, **kwargs): 

131 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

132 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT) 

133 return 

134 

135 @classmethod 

136 def getPackageDir(cls): 

137 return "/path/to/nowhere" 

138 

139 

140def checkCompression(testCase, additionalData): 

141 """Check that compression settings are present 

142 

143 We check that we can access the required settings, and that 

144 the seed is non-zero (zero causes lsst.afw.math.Random to fail). 

145 """ 

146 for plane in ("image", "mask", "variance"): 

147 for entry in ( 

148 "compression.algorithm", 

149 "compression.columns", 

150 "compression.rows", 

151 "compression.quantizeLevel", 

152 "scaling.algorithm", 

153 "scaling.bitpix", 

154 "scaling.maskPlanes", 

155 "scaling.seed", 

156 "scaling.quantizeLevel", 

157 "scaling.quantizePad", 

158 "scaling.fuzz", 

159 "scaling.bscale", 

160 "scaling.bzero", 

161 ): 

162 additionalData.getScalar(plane + "." + entry) 

163 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0) 

164 

165 

166class Mapper1TestCase(unittest.TestCase): 

167 """A test case for the mapper used by the data butler.""" 

168 

169 def setUp(self): 

170 self.mapper = MinMapper1(root=ROOT) 

171 

172 def tearDown(self): 

173 del self.mapper 

174 

175 def testGetDatasetTypes(self): 

176 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

177 # Add the expected additional types to what the base class provides 

178 expectedTypes.extend( 

179 [ 

180 "x", 

181 "x_filename", 

182 "badSourceHist", 

183 "badSourceHist_filename", 

184 ] 

185 ) 

186 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes)) 

187 

188 def testMap(self): 

189 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True) 

190 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

191 self.assertEqual(loc.getCppType(), "BoxI") 

192 self.assertEqual(loc.getStorageName(), "PickleStorage") 

193 expectedRoot = ROOT 

194 expectedLocations = ["foo-1,1.pickle"] 

195 self.assertEqual(loc.getStorage().root, expectedRoot) 

196 self.assertEqual(loc.getLocations(), expectedLocations) 

197 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

198 

199 def testQueryMetadata(self): 

200 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)]) 

201 

202 def testStandardize(self): 

203 self.assertTrue(self.mapper.canStandardize("x")) 

204 self.assertFalse(self.mapper.canStandardize("badSourceHist")) 

205 self.assertFalse(self.mapper.canStandardize("notPresent")) 

206 result = self.mapper.standardize("x", 3, None) 

207 self.assertIsInstance(result, float) 

208 self.assertEqual(result, 3.0) 

209 result = self.mapper.standardize("x", 3.14, None) 

210 self.assertIsInstance(result, float) 

211 self.assertEqual(result, 3.14) 

212 result = self.mapper.standardize("x", "3.14", None) 

213 self.assertIsInstance(result, float) 

214 self.assertEqual(result, 3.14) 

215 

216 def testNames(self): 

217 self.assertEqual(MinMapper1.getCameraName(), "min") 

218 self.assertEqual(MinMapper1.getPackageName(), "larry") 

219 

220 

221class Mapper2TestCase(unittest.TestCase): 

222 """A test case for the mapper used by the data butler.""" 

223 

224 def setUp(self): 

225 super().setUp() 

226 # Force a standard set of filters even for tests that don't use 

227 # MinCam directly. 

228 MinCam() 

229 

230 def testGetDatasetTypes(self): 

231 mapper = MinMapper2(root=ROOT) 

232 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

233 # Add the expected additional types to what the base class provides 

234 expectedTypes.extend( 

235 [ 

236 "flat", 

237 "flat_md", 

238 "flat_filename", 

239 "flat_sub", 

240 "raw", 

241 "raw_md", 

242 "raw_filename", 

243 "raw_sub", 

244 "some", 

245 "some_filename", 

246 "some_md", 

247 "some_sub", 

248 "someCatalog", 

249 "someCatalog_md", 

250 "someCatalog_filename", 

251 "someCatalog_len", 

252 "someCatalog_schema", 

253 "forced_src", 

254 "forced_src_md", 

255 "forced_src_filename", 

256 "forced_src_len", 

257 "forced_src_schema", 

258 "other_sub", 

259 "other_filename", 

260 "other_md", 

261 "other", 

262 "someGz", 

263 "someGz_filename", 

264 "someFz", 

265 "someFz_filename", 

266 "someGz_md", 

267 "someFz_sub", 

268 "someFz_md", 

269 "someGz_sub", 

270 "someGz_bbox", 

271 "someFz_bbox", 

272 "some_bbox", 

273 "other_bbox", 

274 "someExp", 

275 "someExp_filename", 

276 "someExp_md", 

277 "someExp_sub", 

278 "someExp_bbox", 

279 "someExp_filterLabel", 

280 "someExp_photoCalib", 

281 "someExp_visitInfo", 

282 "someExp_detector", 

283 "someExp_filter", 

284 "someExp_header_wcs", 

285 "someExp_wcs", 

286 ] 

287 ) 

288 self.assertEqual(set(mapper.getDatasetTypes()), set(expectedTypes)) 

289 

290 def testMap(self): 

291 mapper = MinMapper2(root=ROOT) 

292 loc = mapper.map("raw", {"ccd": 13}, write=True) 

293 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

294 self.assertEqual(loc.getCppType(), "ImageU") 

295 self.assertEqual(loc.getStorageName(), "FitsStorage") 

296 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

297 self.assertEqual(loc.getStorage().root, ROOT) 

298 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

299 checkCompression(self, loc.getAdditionalData()) 

300 

301 def testSubMap(self): 

302 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

303 mapper = MinMapper2(root=ROOT) 

304 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True) 

305 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

306 self.assertEqual(loc.getCppType(), "ImageU") 

307 self.assertEqual(loc.getStorageName(), "FitsStorage") 

308 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

309 self.assertEqual(loc.getStorage().root, ROOT) 

310 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

311 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

312 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

313 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

314 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

315 checkCompression(self, loc.getAdditionalData()) 

316 

317 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True) 

318 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

319 self.assertEqual(loc.getCppType(), "ImageU") 

320 self.assertEqual(loc.getStorageName(), "FitsStorage") 

321 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

322 self.assertEqual(loc.getStorage().root, ROOT) 

323 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

324 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

325 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

326 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

327 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

328 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT") 

329 checkCompression(self, loc.getAdditionalData()) 

330 

331 def testCatalogExtras(self): 

332 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) 

333 schema = afwTable.Schema() 

334 aa = schema.addField("a", type=np.int32, doc="a") 

335 bb = schema.addField("b", type=np.float64, doc="b") 

336 catalog = lsst.afw.table.BaseCatalog(schema) 

337 row = catalog.addNew() 

338 row.set(aa, 12345) 

339 row.set(bb, 1.2345) 

340 size = len(catalog) 

341 dataId = dict(visit=123, ccd=45) 

342 butler.put(catalog, "someCatalog", dataId) 

343 filename = butler.get("someCatalog_filename", dataId)[0] 

344 try: 

345 self.assertTrue(os.path.exists(filename)) 

346 self.assertEqual(butler.get("someCatalog_schema", dataId), schema) 

347 self.assertEqual(butler.get("someCatalog_len", dataId), size) 

348 header = butler.get("someCatalog_md", dataId) 

349 self.assertEqual(header.getScalar("NAXIS2"), size) 

350 finally: 

351 try: 

352 os.remove(filename) 

353 except OSError as exc: 

354 print("Warning: could not remove file %r: %s" % (filename, exc)) 

355 

356 def testImage(self): 

357 mapper = MinMapper2(root=ROOT) 

358 loc = mapper.map("some", dict(ccd=35)) 

359 expectedLocations = ["bar-35.fits"] 

360 self.assertEqual(loc.getStorage().root, ROOT) 

361 self.assertEqual(loc.getLocations(), expectedLocations) 

362 

363 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

364 image = butler.get("some", ccd=35) 

365 self.assertEqual(image.getFilter().bandLabel, "r") 

366 

367 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox()) 

368 

369 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

370 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

371 self.assertEqual(image.getHeight(), 400) 

372 self.assertEqual(image.getWidth(), 300) 

373 

374 def testFilter(self): 

375 """Test that the same (patched) filter is returned through all Butler 

376 retrieval paths. 

377 """ 

378 mapper = MinMapper2(root=ROOT) 

379 

380 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

381 image = butler.get("someExp", ccd=35) 

382 filter = butler.get("someExp_filter", ccd=35) 

383 # Test only valid with a complete filter 

384 self.assertEqual(image.getFilter(), afwImage.FilterLabel(band="r", physical="r.MP9601")) 

385 # Datasets should give consistent answers 

386 self.assertEqual(filter, image.getFilter()) 

387 

388 def testDetector(self): 

389 mapper = MinMapper2(root=ROOT) 

390 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

391 detector = butler.get("raw_detector", ccd=0) 

392 self.assertEqual(detector.getName(), "ccd00") 

393 

394 def testGzImage(self): 

395 mapper = MinMapper2(root=ROOT) 

396 loc = mapper.map("someGz", dict(ccd=35)) 

397 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")] 

398 self.assertEqual(loc.getStorage().root, ROOT) 

399 self.assertEqual(loc.getLocations(), expectedLocations) 

400 

401 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

402 image = butler.get("someGz", ccd=35) 

403 self.assertEqual(image.getFilter().bandLabel, "r") 

404 

405 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

406 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

407 self.assertEqual(image.getHeight(), 400) 

408 self.assertEqual(image.getWidth(), 300) 

409 

410 def testFzImage(self): 

411 mapper = MinMapper2(root=ROOT) 

412 loc = mapper.map("someFz", dict(ccd=35)) 

413 expectedRoot = ROOT 

414 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")] 

415 self.assertEqual(loc.getStorage().root, expectedRoot) 

416 self.assertEqual(loc.getLocations(), expectedLocations) 

417 

418 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

419 image = butler.get("someFz", ccd=35) 

420 self.assertEqual(image.getFilter().bandLabel, "r") 

421 

422 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

423 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

424 self.assertEqual(image.getHeight(), 400) 

425 self.assertEqual(image.getWidth(), 300) 

426 

427 def testButlerQueryMetadata(self): 

428 mapper = MinMapper2(root=ROOT) 

429 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

430 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, "taiObs": "2005-04-02T09:24:49.933440000"} 

431 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731]) 

432 self.assertEqual( 

433 butler.queryMetadata( 

434 "other", 

435 "visit", 

436 visit=kwargs["visit"], 

437 ccd=kwargs["ccd"], 

438 taiObs=kwargs["taiObs"], 

439 filter=kwargs["filter"], 

440 ), 

441 [787731], 

442 ) 

443 # now test we get no matches if ccd is out of range 

444 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), []) 

445 

446 def testQueryMetadata(self): 

447 mapper = MinMapper2(root=ROOT) 

448 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), [(x,) for x in range(36) if x != 3]) 

449 

450 def testStandardize(self): 

451 mapper = MinMapper2(root=ROOT) 

452 self.assertEqual(mapper.canStandardize("raw"), True) 

453 self.assertEqual(mapper.canStandardize("notPresent"), False) 

454 

455 def testStandardizeFiltersFilterDefs(self): 

456 # tuples are (input, desired output) 

457 testLabels = [ 

458 (None, None), 

459 ( 

460 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

461 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

462 ), 

463 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

464 (afwImage.FilterLabel(physical="i.MP9701"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

465 ( 

466 afwImage.FilterLabel(band="i", physical="old-i"), 

467 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

468 ), 

469 (afwImage.FilterLabel(physical="old-i"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

470 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")), 

471 ] 

472 testIds = [ 

473 {"visit": 12345, "ccd": 42, "filter": f} 

474 for f in { 

475 "i", 

476 "i.MP9701", 

477 "old-i", 

478 "i2", 

479 } 

480 ] 

481 testData = [] 

482 # Resolve special combinations where the expected output is different 

483 for input, corrected in testLabels: 

484 for dataId in testIds: 

485 if input is None: 

486 if dataId["filter"] == "i2": 

487 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

488 else: 

489 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701")) 

490 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2": 

491 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

492 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"): 

493 # Contradictory inputs, leave as-is 

494 data = (input, dataId, input) 

495 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2": 

496 # Contradictory inputs, leave as-is 

497 data = (input, dataId, input) 

498 else: 

499 data = (input, dataId, corrected) 

500 testData.append(data) 

501 

502 mapper = MinMapper2(root=ROOT) 

503 for label, dataId, corrected in testData: 

504 exposure = afwImage.ExposureF() 

505 exposure.setFilter(label) 

506 mapper._setFilter(mapper.exposures["raw"], exposure, dataId) 

507 self.assertEqual(exposure.getFilter(), corrected, msg=f"Started from {label} and {dataId}") 

508 

509 def testStandardizeFiltersFilterNoDefs(self): 

510 testLabels = [ 

511 None, 

512 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

513 afwImage.FilterLabel(band="i"), 

514 afwImage.FilterLabel(physical="i.MP9701"), 

515 afwImage.FilterLabel(band="i", physical="old-i"), 

516 afwImage.FilterLabel(physical="old-i"), 

517 afwImage.FilterLabel(physical="i2"), 

518 ] 

519 testIds = [ 

520 {"visit": 12345, "ccd": 42, "filter": f} 

521 for f in { 

522 "i", 

523 "i.MP9701", 

524 "old-i", 

525 "i2", 

526 } 

527 ] 

528 testData = [] 

529 # Resolve special combinations where the expected output is different 

530 for input in testLabels: 

531 for dataId in testIds: 

532 # FilterLabel is only source of truth if no FilterDefinitions. 

533 data = (input, dataId, input) 

534 testData.append(data) 

535 

536 mapper = MinMapper1(root=ROOT) 

537 for label, dataId, corrected in testData: 

538 exposure = afwImage.ExposureF() 

539 exposure.setFilter(label) 

540 mapper._setFilter(mapper.exposures["raw"], exposure, dataId) 

541 self.assertEqual(exposure.getFilter(), corrected, msg=f"Started from {label} and {dataId}") 

542 

543 def testCalib(self): 

544 mapper = MinMapper2(root=ROOT) 

545 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True) 

546 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF") 

547 self.assertEqual(loc.getCppType(), "ExposureF") 

548 self.assertEqual(loc.getStorageName(), "FitsStorage") 

549 expectedRoot = ROOT 

550 expectedLocations = ["flat-05Am03-fi.fits"] 

551 self.assertEqual(loc.getStorage().root, expectedRoot) 

552 self.assertEqual(loc.getLocations(), expectedLocations) 

553 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

554 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650) 

555 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03") 

556 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i") 

557 checkCompression(self, loc.getAdditionalData()) 

558 

559 def testNames(self): 

560 self.assertEqual(MinMapper2.getCameraName(), "min") 

561 self.assertEqual(MinMapper2.getPackageName(), "moe") 

562 

563 @unittest.expectedFailure 

564 def testParentSearch(self): 

565 mapper = MinMapper2(root=ROOT) 

566 paths = mapper.parentSearch( 

567 os.path.join(ROOT, "testParentSearch"), 

568 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits")), 

569 ) 

570 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits"))]) 

571 paths = mapper.parentSearch( 

572 os.path.join(ROOT, "testParentSearch"), 

573 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]")), 

574 ) 

575 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]"))]) 

576 

577 paths = mapper.parentSearch( 

578 os.path.join(ROOT, "testParentSearch"), 

579 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits")), 

580 ) 

581 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits"))]) 

582 paths = mapper.parentSearch( 

583 os.path.join(ROOT, "testParentSearch"), 

584 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits[1]")), 

585 ) 

586 self.assertEqual( 

587 paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits[1]"))] 

588 ) 

589 

590 def testSkymapLookups(self): 

591 """Test that metadata lookups don't try to get skymap data ID values 

592 from the registry. 

593 """ 

594 mapper = MinMapper2(root=ROOT) 

595 butler = dafPersist.Butler(mapper=mapper) 

596 with self.assertRaises(RuntimeError) as manager: 

597 butler.dataRef("forced_src", visit=787650, ccd=13) 

598 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception)) 

599 # We're mostly concerned that the statements below will raise an 

600 # exception; if they don't, it's not likely the following tests will 

601 # fail. 

602 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0) 

603 self.assertEqual(len(subset), 1) 

604 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0) 

605 self.assertFalse(dataRef.datasetExists("forced_src")) 

606 

607 

608class Mapper3TestCase(unittest.TestCase): 

609 """A test case for a mapper subclass which does not assign packageName.""" 

610 

611 def testPackageName(self): 

612 with self.assertRaises(ValueError): 

613 MinMapper3() 

614 with self.assertRaises(ValueError): 

615 MinMapper3.getPackageName() 

616 

617 

618class ParentRegistryTestCase(unittest.TestCase): 

619 @staticmethod 

620 def _createRegistry(path): 

621 cmd = """CREATE TABLE x( 

622 id INT, 

623 visit INT, 

624 filter TEXT, 

625 snap INT, 

626 raft TEXT, 

627 sensor TEXT, 

628 channel TEXT, 

629 taiObs TEXT, 

630 expTime REAL 

631 ); 

632 """ 

633 conn = sqlite3.connect(path) 

634 conn.cursor().execute(cmd) 

635 conn.commit() 

636 conn.close() 

637 

638 def setUp(self): 

639 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-") 

640 self.repoARoot = os.path.join(self.ROOT, "a") 

641 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1) 

642 butler = dafPersist.Butler(outputs=args) 

643 self._createRegistry(os.path.join(self.repoARoot, "registry.sqlite3")) 

644 del butler 

645 

646 def tearDown(self): 

647 # the butler sql registry closes its database connection in __del__. 

648 # To trigger __del__ we explicitly collect the garbage here. If we 

649 # find having or closing the open database connection is a problem in 

650 # production code, we may need to add api to butler to explicity 

651 # release database connections (and maybe other things like in-memory 

652 # cached objects). 

653 gc.collect() 

654 if os.path.exists(self.ROOT): 

655 shutil.rmtree(self.ROOT) 

656 

657 def test(self): 

658 """Verify that when the child repo does not have a registry it is 

659 assigned the registry from the parent. 

660 """ 

661 repoBRoot = os.path.join(self.ROOT, "b") 

662 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

663 # This way of getting the registry from the mapping is obviously going 

664 # way into private members and the python lambda implementation code. 

665 # It is very brittle and should not be duplicated in user code 

666 # or any location that is not trivial to fix along with changes to the 

667 # CameraMapper or Mapping. 

668 registryA = butler._repos.inputs()[0].repo._mapper.registry 

669 registryB = butler._repos.outputs()[0].repo._mapper.registry 

670 self.assertEqual(id(registryA), id(registryB)) 

671 

672 self._createRegistry(os.path.join(repoBRoot, "registry.sqlite3")) 

673 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

674 # see above; don't copy this way of getting the registry. 

675 registryA = butler._repos.inputs()[0].repo._mapper.registry 

676 registryB = butler._repos.outputs()[0].repo._mapper.registry 

677 self.assertNotEqual(id(registryA), id(registryB)) 

678 

679 

680class MissingPolicyKeyTestCase(unittest.TestCase): 

681 def testGetRaises(self): 

682 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

683 # MinMapper1 does not specify a template for the raw dataset type so 

684 # trying to use it for get should raise 

685 with self.assertRaises(RuntimeError) as contextManager: 

686 butler.get("raw") 

687 # This test demonstrates and verifies that simple use of the incomplete 

688 # dataset type returns a helpful (I hope) error message. 

689 self.assertEqual( 

690 str(contextManager.exception), 

691 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

692 ) 

693 with self.assertRaises(RuntimeError) as contextManager: 

694 butler.queryMetadata("raw", "unused", {}) 

695 

696 def testQueryMetadataRaises(self): 

697 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

698 # MinMapper1 does not specify a template for the raw dataset type so 

699 # trying to use it for queryMetadata should raise 

700 with self.assertRaises(RuntimeError) as contextManager: 

701 butler.queryMetadata("raw", "unused", {}) 

702 # This test demonstrates and verifies that simple use of the incomplete 

703 # dataset type returns a helpful (I hope) error message. 

704 self.assertEqual( 

705 str(contextManager.exception), 

706 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

707 ) 

708 

709 def testFilenameRaises(self): 

710 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

711 # MinMapper1 does not specify a template for the raw dataset type so 

712 # trying to use it for <datasetType>_filename should raise 

713 with self.assertRaises(RuntimeError) as contextManager: 

714 butler.get("raw_filename") 

715 # This test demonstrates and verifies that simple use of the 

716 # incomplete dataset type returns a helpful (I hope) error message. 

717 self.assertEqual( 

718 str(contextManager.exception), 

719 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

720 ) 

721 

722 def testWcsRaises(self): 

723 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

724 # MinMapper1 does not specify a template for the raw dataset type so 

725 # trying to use it for <datasetType>_wcs should raise 

726 with self.assertRaises(RuntimeError) as contextManager: 

727 butler.get("raw_wcs") 

728 # This test demonstrates and verifies that simple use of the 

729 # incomplete dataset type returns a helpful (I hope) error message. 

730 self.assertEqual( 

731 str(contextManager.exception), 

732 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

733 ) 

734 

735 def testConflictRaises(self): 

736 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml")) 

737 with self.assertRaisesRegex(ValueError, r"Duplicate mapping policy for dataset type packages"): 

738 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841 

739 

740 

741class MemoryTester(lsst.utils.tests.MemoryTestCase): 

742 pass 

743 

744 

745if __name__ == "__main__": 745 ↛ 746line 745 didn't jump to line 746, because the condition on line 745 was never true

746 lsst.utils.tests.init() 

747 unittest.main()