Coverage for tests/test_cameraMapper.py: 25%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

404 statements  

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import gc 

23import os 

24import shutil 

25import sqlite3 

26import tempfile 

27import unittest 

28 

29import lsst.afw.image as afwImage 

30import lsst.afw.table as afwTable 

31import lsst.daf.persistence as dafPersist 

32import lsst.geom as geom 

33import lsst.obs.base 

34import lsst.utils.tests 

35import numpy as np 

36from lsst.obs.base.test import BaseMapper 

37 

38ROOT = os.path.abspath(os.path.dirname(__file__)) 

39 

40 

41def setup_module(module): 

42 lsst.utils.tests.init() 

43 

44 

45class MinCam(lsst.obs.base.Instrument): 

46 @property 

47 def filterDefinitions(self): 

48 return lsst.obs.base.FilterDefinitionCollection( 

49 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374), 

50 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487), 

51 lsst.obs.base.FilterDefinition( 

52 physical_filter="r.MP9601", band="r", alias={"old-r"}, lambdaEff=628 

53 ), 

54 lsst.obs.base.FilterDefinition( 

55 physical_filter="i.MP9701", band="i", alias={"old-i"}, lambdaEff=778 

56 ), 

57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170), 

58 # afw_name is so special-cased that only a real example will work 

59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623), 

60 ) 

61 

62 @classmethod 

63 def getName(cls): 

64 return "min" 

65 

66 def getCamera(self): 

67 raise NotImplementedError() 

68 

69 def register(self, registry): 

70 raise NotImplementedError() 

71 

72 def getRawFormatter(self, dataId): 

73 raise NotImplementedError() 

74 

75 def makeDataIdTranslatorFactory(self): 

76 raise NotImplementedError() 

77 

78 

79class MinMapper1(lsst.obs.base.CameraMapper): 

80 packageName = "larry" 

81 

82 def __init__(self, **kwargs): 

83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

85 return 

86 

87 def std_x(self, item, dataId): 

88 return float(item) 

89 

90 @classmethod 

91 def getCameraName(cls): 

92 """Return the name of the camera that this CameraMapper is for.""" 

93 return "min" 

94 

95 @classmethod 

96 def getPackageDir(cls): 

97 return "/path/to/nowhere" 

98 

99 

100class MinMapper2(lsst.obs.base.CameraMapper): 

101 packageName = "moe" 

102 _gen3instrument = MinCam 

103 

104 # CalibRoot in policy 

105 # needCalibRegistry 

106 def __init__(self, **kwargs): 

107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml")) 

108 lsst.obs.base.CameraMapper.__init__( 

109 self, policy=policy, repositoryDir=ROOT, registry="cfhtls.sqlite3", **kwargs 

110 ) 

111 return 

112 

113 def _transformId(self, dataId): 

114 return dataId 

115 

116 def _extractDetectorName(self, dataId): 

117 return "ccd00" 

118 

119 def std_x(self, item, dataId): 

120 return float(item) 

121 

122 @classmethod 

123 def getCameraName(cls): 

124 """Return the name of the camera that this CameraMapper is for.""" 

125 return "min" 

126 

127 @classmethod 

128 def getPackageDir(cls): 

129 return "/path/to/nowhere" 

130 

131 

132# does not assign packageName 

133class MinMapper3(lsst.obs.base.CameraMapper): 

134 def __init__(self, **kwargs): 

135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT) 

137 return 

138 

139 @classmethod 

140 def getPackageDir(cls): 

141 return "/path/to/nowhere" 

142 

143 

144def checkCompression(testCase, additionalData): 

145 """Check that compression settings are present 

146 

147 We check that we can access the required settings, and that 

148 the seed is non-zero (zero causes lsst.afw.math.Random to fail). 

149 """ 

150 for plane in ("image", "mask", "variance"): 

151 for entry in ( 

152 "compression.algorithm", 

153 "compression.columns", 

154 "compression.rows", 

155 "compression.quantizeLevel", 

156 "scaling.algorithm", 

157 "scaling.bitpix", 

158 "scaling.maskPlanes", 

159 "scaling.seed", 

160 "scaling.quantizeLevel", 

161 "scaling.quantizePad", 

162 "scaling.fuzz", 

163 "scaling.bscale", 

164 "scaling.bzero", 

165 ): 

166 additionalData.getScalar(plane + "." + entry) 

167 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0) 

168 

169 

170class Mapper1TestCase(unittest.TestCase): 

171 """A test case for the mapper used by the data butler.""" 

172 

173 def setUp(self): 

174 self.mapper = MinMapper1(root=ROOT) 

175 

176 def tearDown(self): 

177 del self.mapper 

178 

179 def testGetDatasetTypes(self): 

180 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

181 # Add the expected additional types to what the base class provides 

182 expectedTypes.extend( 

183 [ 

184 "x", 

185 "x_filename", 

186 "badSourceHist", 

187 "badSourceHist_filename", 

188 ] 

189 ) 

190 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes)) 

191 

192 def testMap(self): 

193 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True) 

194 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

195 self.assertEqual(loc.getCppType(), "BoxI") 

196 self.assertEqual(loc.getStorageName(), "PickleStorage") 

197 expectedRoot = ROOT 

198 expectedLocations = ["foo-1,1.pickle"] 

199 self.assertEqual(loc.getStorage().root, expectedRoot) 

200 self.assertEqual(loc.getLocations(), expectedLocations) 

201 self.assertEqual(loc.getAdditionalData().toString(), 'sensor = "1,1"\n') 

202 

203 def testQueryMetadata(self): 

204 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)]) 

205 

206 def testStandardize(self): 

207 self.assertTrue(self.mapper.canStandardize("x")) 

208 self.assertFalse(self.mapper.canStandardize("badSourceHist")) 

209 self.assertFalse(self.mapper.canStandardize("notPresent")) 

210 result = self.mapper.standardize("x", 3, None) 

211 self.assertIsInstance(result, float) 

212 self.assertEqual(result, 3.0) 

213 result = self.mapper.standardize("x", 3.14, None) 

214 self.assertIsInstance(result, float) 

215 self.assertEqual(result, 3.14) 

216 result = self.mapper.standardize("x", "3.14", None) 

217 self.assertIsInstance(result, float) 

218 self.assertEqual(result, 3.14) 

219 

220 def testNames(self): 

221 self.assertEqual(MinMapper1.getCameraName(), "min") 

222 self.assertEqual(MinMapper1.getPackageName(), "larry") 

223 

224 

225class Mapper2TestCase(unittest.TestCase): 

226 """A test case for the mapper used by the data butler.""" 

227 

228 def setUp(self): 

229 super().setUp() 

230 # Force a standard set of filters even for tests that don't use 

231 # MinCam directly. 

232 MinCam() 

233 

234 def testGetDatasetTypes(self): 

235 mapper = MinMapper2(root=ROOT) 

236 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

237 # Add the expected additional types to what the base class provides 

238 expectedTypes.extend( 

239 [ 

240 "flat", 

241 "flat_md", 

242 "flat_filename", 

243 "flat_sub", 

244 "raw", 

245 "raw_md", 

246 "raw_filename", 

247 "raw_sub", 

248 "some", 

249 "some_filename", 

250 "some_md", 

251 "some_sub", 

252 "someCatalog", 

253 "someCatalog_md", 

254 "someCatalog_filename", 

255 "someCatalog_len", 

256 "someCatalog_schema", 

257 "forced_src", 

258 "forced_src_md", 

259 "forced_src_filename", 

260 "forced_src_len", 

261 "forced_src_schema", 

262 "other_sub", 

263 "other_filename", 

264 "other_md", 

265 "other", 

266 "someGz", 

267 "someGz_filename", 

268 "someFz", 

269 "someFz_filename", 

270 "someGz_md", 

271 "someFz_sub", 

272 "someFz_md", 

273 "someGz_sub", 

274 "someGz_bbox", 

275 "someFz_bbox", 

276 "some_bbox", 

277 "other_bbox", 

278 "someExp", 

279 "someExp_filename", 

280 "someExp_md", 

281 "someExp_sub", 

282 "someExp_bbox", 

283 "someExp_filterLabel", 

284 "someExp_photoCalib", 

285 "someExp_visitInfo", 

286 "someExp_detector", 

287 "someExp_filter", 

288 "someExp_header_wcs", 

289 "someExp_wcs", 

290 ] 

291 ) 

292 self.assertEqual(set(mapper.getDatasetTypes()), set(expectedTypes)) 

293 

294 def testMap(self): 

295 mapper = MinMapper2(root=ROOT) 

296 loc = mapper.map("raw", {"ccd": 13}, write=True) 

297 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

298 self.assertEqual(loc.getCppType(), "ImageU") 

299 self.assertEqual(loc.getStorageName(), "FitsStorage") 

300 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

301 self.assertEqual(loc.getStorage().root, ROOT) 

302 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

303 checkCompression(self, loc.getAdditionalData()) 

304 

305 def testSubMap(self): 

306 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

307 mapper = MinMapper2(root=ROOT) 

308 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True) 

309 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

310 self.assertEqual(loc.getCppType(), "ImageU") 

311 self.assertEqual(loc.getStorageName(), "FitsStorage") 

312 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

313 self.assertEqual(loc.getStorage().root, ROOT) 

314 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

315 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

316 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

317 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

318 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

319 checkCompression(self, loc.getAdditionalData()) 

320 

321 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True) 

322 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

323 self.assertEqual(loc.getCppType(), "ImageU") 

324 self.assertEqual(loc.getStorageName(), "FitsStorage") 

325 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

326 self.assertEqual(loc.getStorage().root, ROOT) 

327 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

328 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

329 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

330 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

331 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

332 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT") 

333 checkCompression(self, loc.getAdditionalData()) 

334 

335 def testCatalogExtras(self): 

336 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) 

337 schema = afwTable.Schema() 

338 aa = schema.addField("a", type=np.int32, doc="a") 

339 bb = schema.addField("b", type=np.float64, doc="b") 

340 catalog = lsst.afw.table.BaseCatalog(schema) 

341 row = catalog.addNew() 

342 row.set(aa, 12345) 

343 row.set(bb, 1.2345) 

344 size = len(catalog) 

345 dataId = dict(visit=123, ccd=45) 

346 butler.put(catalog, "someCatalog", dataId) 

347 filename = butler.get("someCatalog_filename", dataId)[0] 

348 try: 

349 self.assertTrue(os.path.exists(filename)) 

350 self.assertEqual(butler.get("someCatalog_schema", dataId), schema) 

351 self.assertEqual(butler.get("someCatalog_len", dataId), size) 

352 header = butler.get("someCatalog_md", dataId) 

353 self.assertEqual(header.getScalar("NAXIS2"), size) 

354 finally: 

355 try: 

356 os.remove(filename) 

357 except OSError as exc: 

358 print("Warning: could not remove file %r: %s" % (filename, exc)) 

359 

360 def testImage(self): 

361 mapper = MinMapper2(root=ROOT) 

362 loc = mapper.map("some", dict(ccd=35)) 

363 expectedLocations = ["bar-35.fits"] 

364 self.assertEqual(loc.getStorage().root, ROOT) 

365 self.assertEqual(loc.getLocations(), expectedLocations) 

366 

367 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

368 image = butler.get("some", ccd=35) 

369 self.assertEqual(image.getFilter().getName(), "r") 

370 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

371 

372 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox()) 

373 

374 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

375 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

376 self.assertEqual(image.getHeight(), 400) 

377 self.assertEqual(image.getWidth(), 300) 

378 

379 def testFilter(self): 

380 """Test that the same (patched) filter is returned through all Butler 

381 retrieval paths. 

382 """ 

383 mapper = MinMapper2(root=ROOT) 

384 

385 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

386 image = butler.get("someExp", ccd=35) 

387 filter = butler.get("someExp_filterLabel", ccd=35) 

388 # Test only valid with a complete filter 

389 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601")) 

390 # Datasets should give consistent answers 

391 self.assertEqual(filter, image.getFilterLabel()) 

392 

393 def testDetector(self): 

394 mapper = MinMapper2(root=ROOT) 

395 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

396 detector = butler.get("raw_detector", ccd=0) 

397 self.assertEqual(detector.getName(), "ccd00") 

398 

399 def testGzImage(self): 

400 mapper = MinMapper2(root=ROOT) 

401 loc = mapper.map("someGz", dict(ccd=35)) 

402 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")] 

403 self.assertEqual(loc.getStorage().root, ROOT) 

404 self.assertEqual(loc.getLocations(), expectedLocations) 

405 

406 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

407 image = butler.get("someGz", ccd=35) 

408 self.assertEqual(image.getFilter().getName(), "r") 

409 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

410 

411 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

412 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

413 self.assertEqual(image.getHeight(), 400) 

414 self.assertEqual(image.getWidth(), 300) 

415 

416 def testFzImage(self): 

417 mapper = MinMapper2(root=ROOT) 

418 loc = mapper.map("someFz", dict(ccd=35)) 

419 expectedRoot = ROOT 

420 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")] 

421 self.assertEqual(loc.getStorage().root, expectedRoot) 

422 self.assertEqual(loc.getLocations(), expectedLocations) 

423 

424 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

425 image = butler.get("someFz", ccd=35) 

426 self.assertEqual(image.getFilter().getName(), "r") 

427 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

428 

429 bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400)) 

430 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

431 self.assertEqual(image.getHeight(), 400) 

432 self.assertEqual(image.getWidth(), 300) 

433 

434 def testButlerQueryMetadata(self): 

435 mapper = MinMapper2(root=ROOT) 

436 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

437 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, "taiObs": "2005-04-02T09:24:49.933440000"} 

438 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731]) 

439 self.assertEqual( 

440 butler.queryMetadata( 

441 "other", 

442 "visit", 

443 visit=kwargs["visit"], 

444 ccd=kwargs["ccd"], 

445 taiObs=kwargs["taiObs"], 

446 filter=kwargs["filter"], 

447 ), 

448 [787731], 

449 ) 

450 # now test we get no matches if ccd is out of range 

451 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), []) 

452 

453 def testQueryMetadata(self): 

454 mapper = MinMapper2(root=ROOT) 

455 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), [(x,) for x in range(36) if x != 3]) 

456 

457 def testStandardize(self): 

458 mapper = MinMapper2(root=ROOT) 

459 self.assertEqual(mapper.canStandardize("raw"), True) 

460 self.assertEqual(mapper.canStandardize("notPresent"), False) 

461 

462 def testStandardizeFiltersFilterDefs(self): 

463 # tuples are (input, desired output) 

464 testLabels = [ 

465 (None, None), 

466 ( 

467 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

468 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

469 ), 

470 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

471 (afwImage.FilterLabel(physical="i.MP9701"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

472 ( 

473 afwImage.FilterLabel(band="i", physical="old-i"), 

474 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

475 ), 

476 (afwImage.FilterLabel(physical="old-i"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

477 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")), 

478 ] 

479 testIds = [ 

480 {"visit": 12345, "ccd": 42, "filter": f} 

481 for f in { 

482 "i", 

483 "i.MP9701", 

484 "old-i", 

485 "i2", 

486 } 

487 ] 

488 testData = [] 

489 # Resolve special combinations where the expected output is different 

490 for input, corrected in testLabels: 

491 for dataId in testIds: 

492 if input is None: 

493 if dataId["filter"] == "i2": 

494 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

495 else: 

496 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701")) 

497 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2": 

498 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

499 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"): 

500 # Contradictory inputs, leave as-is 

501 data = (input, dataId, input) 

502 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2": 

503 # Contradictory inputs, leave as-is 

504 data = (input, dataId, input) 

505 else: 

506 data = (input, dataId, corrected) 

507 testData.append(data) 

508 

509 mapper = MinMapper2(root=ROOT) 

510 for label, dataId, corrected in testData: 

511 exposure = afwImage.ExposureF() 

512 exposure.setFilterLabel(label) 

513 mapper._setFilter(mapper.exposures["raw"], exposure, dataId) 

514 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

515 

516 def testStandardizeFiltersFilterNoDefs(self): 

517 testLabels = [ 

518 None, 

519 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

520 afwImage.FilterLabel(band="i"), 

521 afwImage.FilterLabel(physical="i.MP9701"), 

522 afwImage.FilterLabel(band="i", physical="old-i"), 

523 afwImage.FilterLabel(physical="old-i"), 

524 afwImage.FilterLabel(physical="i2"), 

525 ] 

526 testIds = [ 

527 {"visit": 12345, "ccd": 42, "filter": f} 

528 for f in { 

529 "i", 

530 "i.MP9701", 

531 "old-i", 

532 "i2", 

533 } 

534 ] 

535 testData = [] 

536 # Resolve special combinations where the expected output is different 

537 for input in testLabels: 

538 for dataId in testIds: 

539 if input is None: 

540 # Can still get some filter info out of the Filter registry 

541 if dataId["filter"] == "i2": 

542 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

543 else: 

544 # Data ID maps to filter(s) with aliases; can't 

545 # unambiguously determine physical filter. 

546 data = (input, dataId, afwImage.FilterLabel(band="i")) 

547 else: 

548 data = (input, dataId, input) 

549 testData.append(data) 

550 

551 mapper = MinMapper1(root=ROOT) 

552 for label, dataId, corrected in testData: 

553 exposure = afwImage.ExposureF() 

554 exposure.setFilterLabel(label) 

555 mapper._setFilter(mapper.exposures["raw"], exposure, dataId) 

556 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

557 

558 def testCalib(self): 

559 mapper = MinMapper2(root=ROOT) 

560 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True) 

561 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF") 

562 self.assertEqual(loc.getCppType(), "ExposureF") 

563 self.assertEqual(loc.getStorageName(), "FitsStorage") 

564 expectedRoot = ROOT 

565 expectedLocations = ["flat-05Am03-fi.fits"] 

566 self.assertEqual(loc.getStorage().root, expectedRoot) 

567 self.assertEqual(loc.getLocations(), expectedLocations) 

568 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

569 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650) 

570 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03") 

571 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i") 

572 checkCompression(self, loc.getAdditionalData()) 

573 

574 def testNames(self): 

575 self.assertEqual(MinMapper2.getCameraName(), "min") 

576 self.assertEqual(MinMapper2.getPackageName(), "moe") 

577 

578 @unittest.expectedFailure 

579 def testParentSearch(self): 

580 mapper = MinMapper2(root=ROOT) 

581 paths = mapper.parentSearch( 

582 os.path.join(ROOT, "testParentSearch"), 

583 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits")), 

584 ) 

585 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits"))]) 

586 paths = mapper.parentSearch( 

587 os.path.join(ROOT, "testParentSearch"), 

588 os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]")), 

589 ) 

590 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "bar.fits[1]"))]) 

591 

592 paths = mapper.parentSearch( 

593 os.path.join(ROOT, "testParentSearch"), 

594 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits")), 

595 ) 

596 self.assertEqual(paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits"))]) 

597 paths = mapper.parentSearch( 

598 os.path.join(ROOT, "testParentSearch"), 

599 os.path.join(ROOT, os.path.join("testParentSearch", "baz.fits[1]")), 

600 ) 

601 self.assertEqual( 

602 paths, [os.path.join(ROOT, os.path.join("testParentSearch", "_parent", "baz.fits[1]"))] 

603 ) 

604 

605 def testSkymapLookups(self): 

606 """Test that metadata lookups don't try to get skymap data ID values 

607 from the registry. 

608 """ 

609 mapper = MinMapper2(root=ROOT) 

610 butler = dafPersist.Butler(mapper=mapper) 

611 with self.assertRaises(RuntimeError) as manager: 

612 butler.dataRef("forced_src", visit=787650, ccd=13) 

613 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception)) 

614 # We're mostly concerned that the statements below will raise an 

615 # exception; if they don't, it's not likely the following tests will 

616 # fail. 

617 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0) 

618 self.assertEqual(len(subset), 1) 

619 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0) 

620 self.assertFalse(dataRef.datasetExists("forced_src")) 

621 

622 

623class Mapper3TestCase(unittest.TestCase): 

624 """A test case for a mapper subclass which does not assign packageName.""" 

625 

626 def testPackageName(self): 

627 with self.assertRaises(ValueError): 

628 MinMapper3() 

629 with self.assertRaises(ValueError): 

630 MinMapper3.getPackageName() 

631 

632 

633class ParentRegistryTestCase(unittest.TestCase): 

634 @staticmethod 

635 def _createRegistry(path): 

636 cmd = """CREATE TABLE x( 

637 id INT, 

638 visit INT, 

639 filter TEXT, 

640 snap INT, 

641 raft TEXT, 

642 sensor TEXT, 

643 channel TEXT, 

644 taiObs TEXT, 

645 expTime REAL 

646 ); 

647 """ 

648 conn = sqlite3.connect(path) 

649 conn.cursor().execute(cmd) 

650 conn.commit() 

651 conn.close() 

652 

653 def setUp(self): 

654 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-") 

655 self.repoARoot = os.path.join(self.ROOT, "a") 

656 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1) 

657 butler = dafPersist.Butler(outputs=args) 

658 self._createRegistry(os.path.join(self.repoARoot, "registry.sqlite3")) 

659 del butler 

660 

661 def tearDown(self): 

662 # the butler sql registry closes its database connection in __del__. 

663 # To trigger __del__ we explicitly collect the garbage here. If we 

664 # find having or closing the open database connection is a problem in 

665 # production code, we may need to add api to butler to explicity 

666 # release database connections (and maybe other things like in-memory 

667 # cached objects). 

668 gc.collect() 

669 if os.path.exists(self.ROOT): 

670 shutil.rmtree(self.ROOT) 

671 

672 def test(self): 

673 """Verify that when the child repo does not have a registry it is 

674 assigned the registry from the parent. 

675 """ 

676 repoBRoot = os.path.join(self.ROOT, "b") 

677 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

678 # This way of getting the registry from the mapping is obviously going 

679 # way into private members and the python lambda implementation code. 

680 # It is very brittle and should not be duplicated in user code 

681 # or any location that is not trivial to fix along with changes to the 

682 # CameraMapper or Mapping. 

683 registryA = butler._repos.inputs()[0].repo._mapper.registry 

684 registryB = butler._repos.outputs()[0].repo._mapper.registry 

685 self.assertEqual(id(registryA), id(registryB)) 

686 

687 self._createRegistry(os.path.join(repoBRoot, "registry.sqlite3")) 

688 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

689 # see above; don't copy this way of getting the registry. 

690 registryA = butler._repos.inputs()[0].repo._mapper.registry 

691 registryB = butler._repos.outputs()[0].repo._mapper.registry 

692 self.assertNotEqual(id(registryA), id(registryB)) 

693 

694 

695class MissingPolicyKeyTestCase(unittest.TestCase): 

696 def testGetRaises(self): 

697 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

698 # MinMapper1 does not specify a template for the raw dataset type so 

699 # trying to use it for get should raise 

700 with self.assertRaises(RuntimeError) as contextManager: 

701 butler.get("raw") 

702 # This test demonstrates and verifies that simple use of the incomplete 

703 # dataset type returns a helpful (I hope) error message. 

704 self.assertEqual( 

705 str(contextManager.exception), 

706 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

707 ) 

708 with self.assertRaises(RuntimeError) as contextManager: 

709 butler.queryMetadata("raw", "unused", {}) 

710 

711 def testQueryMetadataRaises(self): 

712 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

713 # MinMapper1 does not specify a template for the raw dataset type so 

714 # trying to use it for queryMetadata should raise 

715 with self.assertRaises(RuntimeError) as contextManager: 

716 butler.queryMetadata("raw", "unused", {}) 

717 # This test demonstrates and verifies that simple use of the incomplete 

718 # dataset type returns a helpful (I hope) error message. 

719 self.assertEqual( 

720 str(contextManager.exception), 

721 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

722 ) 

723 

724 def testFilenameRaises(self): 

725 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

726 # MinMapper1 does not specify a template for the raw dataset type so 

727 # trying to use it for <datasetType>_filename should raise 

728 with self.assertRaises(RuntimeError) as contextManager: 

729 butler.get("raw_filename") 

730 # This test demonstrates and verifies that simple use of the 

731 # incomplete dataset type returns a helpful (I hope) error message. 

732 self.assertEqual( 

733 str(contextManager.exception), 

734 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

735 ) 

736 

737 def testWcsRaises(self): 

738 butler = dafPersist.Butler(inputs={"root": ROOT, "mapper": MinMapper1}) 

739 # MinMapper1 does not specify a template for the raw dataset type so 

740 # trying to use it for <datasetType>_wcs should raise 

741 with self.assertRaises(RuntimeError) as contextManager: 

742 butler.get("raw_wcs") 

743 # This test demonstrates and verifies that simple use of the 

744 # incomplete dataset type returns a helpful (I hope) error message. 

745 self.assertEqual( 

746 str(contextManager.exception), 

747 "Template is not defined for the raw dataset type, it must be set before it can be used.", 

748 ) 

749 

750 def testConflictRaises(self): 

751 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml")) 

752 with self.assertRaisesRegex(ValueError, r"Duplicate mapping policy for dataset type packages"): 

753 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841 

754 

755 

756class MemoryTester(lsst.utils.tests.MemoryTestCase): 

757 pass 

758 

759 

760if __name__ == "__main__": 760 ↛ 761line 760 didn't jump to line 761, because the condition on line 760 was never true

761 lsst.utils.tests.init() 

762 unittest.main()