Coverage for tests/test_cameraMapper.py: 22%

404 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:01 -0800

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import gc 

23import os 

24import sqlite3 

25import unittest 

26import tempfile 

27 

28import numpy as np 

29 

30import lsst.utils.tests 

31import lsst.geom as geom 

32import lsst.afw.image as afwImage 

33import lsst.afw.table as afwTable 

34import lsst.daf.persistence as dafPersist 

35import lsst.obs.base 

36import shutil 

37 

38from lsst.obs.base.test import BaseMapper 

39 

40ROOT = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43def setup_module(module): 

44 lsst.utils.tests.init() 

45 

46 

47class MinCam(lsst.obs.base.Instrument): 

48 @property 

49 def filterDefinitions(self): 

50 return lsst.obs.base.FilterDefinitionCollection( 

51 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374), 

52 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487), 

53 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"}, 

54 lambdaEff=628), 

55 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"}, 

56 lambdaEff=778), 

57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170), 

58 # afw_name is so special-cased that only a real example will work 

59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623), 

60 ) 

61 

62 @classmethod 

63 def getName(cls): 

64 return "min" 

65 

66 def getCamera(self): 

67 raise NotImplementedError() 

68 

69 def register(self, registry): 

70 raise NotImplementedError() 

71 

72 def getRawFormatter(self, dataId): 

73 raise NotImplementedError() 

74 

75 def makeDataIdTranslatorFactory(self): 

76 raise NotImplementedError() 

77 

78 

79class MinMapper1(lsst.obs.base.CameraMapper): 

80 packageName = 'larry' 

81 

82 def __init__(self, **kwargs): 

83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

85 return 

86 

87 def std_x(self, item, dataId): 

88 return float(item) 

89 

90 @classmethod 

91 def getCameraName(cls): 

92 """Return the name of the camera that this CameraMapper is for.""" 

93 return "min" 

94 

95 @classmethod 

96 def getPackageDir(cls): 

97 return "/path/to/nowhere" 

98 

99 

100class MinMapper2(lsst.obs.base.CameraMapper): 

101 packageName = 'moe' 

102 _gen3instrument = MinCam 

103 

104 # CalibRoot in policy 

105 # needCalibRegistry 

106 def __init__(self, **kwargs): 

107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml")) 

108 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, 

109 registry="cfhtls.sqlite3", **kwargs) 

110 return 

111 

112 def _transformId(self, dataId): 

113 return dataId 

114 

115 def _extractDetectorName(self, dataId): 

116 return "ccd00" 

117 

118 def std_x(self, item, dataId): 

119 return float(item) 

120 

121 @classmethod 

122 def getCameraName(cls): 

123 """Return the name of the camera that this CameraMapper is for.""" 

124 return "min" 

125 

126 @classmethod 

127 def getPackageDir(cls): 

128 return "/path/to/nowhere" 

129 

130 

131# does not assign packageName 

132class MinMapper3(lsst.obs.base.CameraMapper): 

133 

134 def __init__(self, **kwargs): 

135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT) 

137 return 

138 

139 @classmethod 

140 def getPackageDir(cls): 

141 return "/path/to/nowhere" 

142 

143 

144def checkCompression(testCase, additionalData): 

145 """Check that compression settings are present 

146 

147 We check that we can access the required settings, and that 

148 the seed is non-zero (zero causes lsst.afw.math.Random to fail). 

149 """ 

150 for plane in ("image", "mask", "variance"): 

151 for entry in ("compression.algorithm", 

152 "compression.columns", 

153 "compression.rows", 

154 "compression.quantizeLevel", 

155 "scaling.algorithm", 

156 "scaling.bitpix", 

157 "scaling.maskPlanes", 

158 "scaling.seed", 

159 "scaling.quantizeLevel", 

160 "scaling.quantizePad", 

161 "scaling.fuzz", 

162 "scaling.bscale", 

163 "scaling.bzero", 

164 ): 

165 additionalData.getScalar(plane + "." + entry) 

166 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0) 

167 

168 

169class Mapper1TestCase(unittest.TestCase): 

170 """A test case for the mapper used by the data butler.""" 

171 

172 def setUp(self): 

173 self.mapper = MinMapper1(root=ROOT) 

174 

175 def tearDown(self): 

176 del self.mapper 

177 

178 def testGetDatasetTypes(self): 

179 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

180 # Add the expected additional types to what the base class provides 

181 expectedTypes.extend(["x", "x_filename", 

182 "badSourceHist", "badSourceHist_filename", ]) 

183 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes)) 

184 

185 def testMap(self): 

186 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True) 

187 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

188 self.assertEqual(loc.getCppType(), "BoxI") 

189 self.assertEqual(loc.getStorageName(), "PickleStorage") 

190 expectedRoot = ROOT 

191 expectedLocations = ["foo-1,1.pickle"] 

192 self.assertEqual(loc.getStorage().root, expectedRoot) 

193 self.assertEqual(loc.getLocations(), expectedLocations) 

194 self.assertEqual(loc.getAdditionalData().toString(), 

195 "sensor = \"1,1\"\n") 

196 

197 def testQueryMetadata(self): 

198 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)]) 

199 

200 def testStandardize(self): 

201 self.assertTrue(self.mapper.canStandardize("x")) 

202 self.assertFalse(self.mapper.canStandardize("badSourceHist")) 

203 self.assertFalse(self.mapper.canStandardize("notPresent")) 

204 result = self.mapper.standardize("x", 3, None) 

205 self.assertIsInstance(result, float) 

206 self.assertEqual(result, 3.0) 

207 result = self.mapper.standardize("x", 3.14, None) 

208 self.assertIsInstance(result, float) 

209 self.assertEqual(result, 3.14) 

210 result = self.mapper.standardize("x", "3.14", None) 

211 self.assertIsInstance(result, float) 

212 self.assertEqual(result, 3.14) 

213 

214 def testNames(self): 

215 self.assertEqual(MinMapper1.getCameraName(), "min") 

216 self.assertEqual(MinMapper1.getPackageName(), "larry") 

217 

218 

219class Mapper2TestCase(unittest.TestCase): 

220 """A test case for the mapper used by the data butler.""" 

221 

222 def setUp(self): 

223 super().setUp() 

224 # Force a standard set of filters even for tests that don't use 

225 # MinCam directly. 

226 MinCam() 

227 

228 def testGetDatasetTypes(self): 

229 mapper = MinMapper2(root=ROOT) 

230 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

231 # Add the expected additional types to what the base class provides 

232 expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub", 

233 "raw", "raw_md", "raw_filename", "raw_sub", 

234 "some", "some_filename", "some_md", "some_sub", 

235 "someCatalog", "someCatalog_md", "someCatalog_filename", 

236 "someCatalog_len", "someCatalog_schema", 

237 "forced_src", "forced_src_md", "forced_src_filename", 

238 "forced_src_len", "forced_src_schema", 

239 "other_sub", "other_filename", "other_md", "other", 

240 "someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md", 

241 "someFz_sub", "someFz_md", "someGz_sub", 

242 "someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox", 

243 "someExp", "someExp_filename", "someExp_md", "someExp_sub", 

244 "someExp_bbox", "someExp_filterLabel", "someExp_photoCalib", 

245 "someExp_visitInfo", "someExp_detector", "someExp_filter", 

246 "someExp_header_wcs", "someExp_wcs", 

247 ]) 

248 self.assertEqual(set(mapper.getDatasetTypes()), 

249 set(expectedTypes)) 

250 

251 def testMap(self): 

252 mapper = MinMapper2(root=ROOT) 

253 loc = mapper.map("raw", {"ccd": 13}, write=True) 

254 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

255 self.assertEqual(loc.getCppType(), "ImageU") 

256 self.assertEqual(loc.getStorageName(), "FitsStorage") 

257 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

258 self.assertEqual(loc.getStorage().root, ROOT) 

259 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

260 checkCompression(self, loc.getAdditionalData()) 

261 

262 def testSubMap(self): 

263 bbox = geom.BoxI(geom.Point2I(200, 100), 

264 geom.Extent2I(300, 400)) 

265 mapper = MinMapper2(root=ROOT) 

266 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True) 

267 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

268 self.assertEqual(loc.getCppType(), "ImageU") 

269 self.assertEqual(loc.getStorageName(), "FitsStorage") 

270 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

271 self.assertEqual(loc.getStorage().root, ROOT) 

272 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

273 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

274 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

275 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

276 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

277 checkCompression(self, loc.getAdditionalData()) 

278 

279 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True) 

280 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

281 self.assertEqual(loc.getCppType(), "ImageU") 

282 self.assertEqual(loc.getStorageName(), "FitsStorage") 

283 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

284 self.assertEqual(loc.getStorage().root, ROOT) 

285 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

286 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

287 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

288 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

289 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

290 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT") 

291 checkCompression(self, loc.getAdditionalData()) 

292 

293 def testCatalogExtras(self): 

294 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) 

295 schema = afwTable.Schema() 

296 aa = schema.addField("a", type=np.int32, doc="a") 

297 bb = schema.addField("b", type=np.float64, doc="b") 

298 catalog = lsst.afw.table.BaseCatalog(schema) 

299 row = catalog.addNew() 

300 row.set(aa, 12345) 

301 row.set(bb, 1.2345) 

302 size = len(catalog) 

303 dataId = dict(visit=123, ccd=45) 

304 butler.put(catalog, "someCatalog", dataId) 

305 filename = butler.get("someCatalog_filename", dataId)[0] 

306 try: 

307 self.assertTrue(os.path.exists(filename)) 

308 self.assertEqual(butler.get("someCatalog_schema", dataId), schema) 

309 self.assertEqual(butler.get("someCatalog_len", dataId), size) 

310 header = butler.get("someCatalog_md", dataId) 

311 self.assertEqual(header.getScalar("NAXIS2"), size) 

312 finally: 

313 try: 

314 os.remove(filename) 

315 except OSError as exc: 

316 print("Warning: could not remove file %r: %s" % (filename, exc)) 

317 

318 def testImage(self): 

319 mapper = MinMapper2(root=ROOT) 

320 loc = mapper.map("some", dict(ccd=35)) 

321 expectedLocations = ["bar-35.fits"] 

322 self.assertEqual(loc.getStorage().root, ROOT) 

323 self.assertEqual(loc.getLocations(), expectedLocations) 

324 

325 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

326 image = butler.get("some", ccd=35) 

327 self.assertEqual(image.getFilter().getName(), "r") 

328 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

329 

330 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox()) 

331 

332 bbox = geom.BoxI(geom.Point2I(200, 100), 

333 geom.Extent2I(300, 400)) 

334 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

335 self.assertEqual(image.getHeight(), 400) 

336 self.assertEqual(image.getWidth(), 300) 

337 

338 def testFilter(self): 

339 """Test that the same (patched) filter is returned through all Butler 

340 retrieval paths. 

341 """ 

342 mapper = MinMapper2(root=ROOT) 

343 

344 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

345 image = butler.get("someExp", ccd=35) 

346 filter = butler.get("someExp_filterLabel", ccd=35) 

347 # Test only valid with a complete filter 

348 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601")) 

349 # Datasets should give consistent answers 

350 self.assertEqual(filter, image.getFilterLabel()) 

351 

352 def testDetector(self): 

353 mapper = MinMapper2(root=ROOT) 

354 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

355 detector = butler.get("raw_detector", ccd=0) 

356 self.assertEqual(detector.getName(), "ccd00") 

357 

358 def testGzImage(self): 

359 mapper = MinMapper2(root=ROOT) 

360 loc = mapper.map("someGz", dict(ccd=35)) 

361 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")] 

362 self.assertEqual(loc.getStorage().root, ROOT) 

363 self.assertEqual(loc.getLocations(), expectedLocations) 

364 

365 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

366 image = butler.get("someGz", ccd=35) 

367 self.assertEqual(image.getFilter().getName(), "r") 

368 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

369 

370 bbox = geom.BoxI(geom.Point2I(200, 100), 

371 geom.Extent2I(300, 400)) 

372 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

373 self.assertEqual(image.getHeight(), 400) 

374 self.assertEqual(image.getWidth(), 300) 

375 

376 def testFzImage(self): 

377 mapper = MinMapper2(root=ROOT) 

378 loc = mapper.map("someFz", dict(ccd=35)) 

379 expectedRoot = ROOT 

380 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")] 

381 self.assertEqual(loc.getStorage().root, expectedRoot) 

382 self.assertEqual(loc.getLocations(), expectedLocations) 

383 

384 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

385 image = butler.get("someFz", ccd=35) 

386 self.assertEqual(image.getFilter().getName(), "r") 

387 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

388 

389 bbox = geom.BoxI(geom.Point2I(200, 100), 

390 geom.Extent2I(300, 400)) 

391 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

392 self.assertEqual(image.getHeight(), 400) 

393 self.assertEqual(image.getWidth(), 300) 

394 

395 def testButlerQueryMetadata(self): 

396 mapper = MinMapper2(root=ROOT) 

397 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

398 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, 

399 "taiObs": "2005-04-02T09:24:49.933440000"} 

400 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731]) 

401 self.assertEqual(butler.queryMetadata("other", "visit", 

402 visit=kwargs["visit"], ccd=kwargs["ccd"], 

403 taiObs=kwargs["taiObs"], filter=kwargs["filter"]), 

404 [787731]) 

405 # now test we get no matches if ccd is out of range 

406 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), []) 

407 

408 def testQueryMetadata(self): 

409 mapper = MinMapper2(root=ROOT) 

410 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), 

411 [(x,) for x in range(36) if x != 3]) 

412 

413 def testStandardize(self): 

414 mapper = MinMapper2(root=ROOT) 

415 self.assertEqual(mapper.canStandardize("raw"), True) 

416 self.assertEqual(mapper.canStandardize("notPresent"), False) 

417 

418 def testStandardizeFiltersFilterDefs(self): 

419 # tuples are (input, desired output) 

420 testLabels = [ 

421 (None, None), 

422 (afwImage.FilterLabel(band="i", physical="i.MP9701"), 

423 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

424 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="i", physical="i.MP9701")), 

425 (afwImage.FilterLabel(physical="i.MP9701"), 

426 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

427 (afwImage.FilterLabel(band="i", physical="old-i"), 

428 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

429 (afwImage.FilterLabel(physical="old-i"), 

430 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

431 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")), 

432 ] 

433 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

434 "i", "i.MP9701", "old-i", "i2", 

435 }] 

436 testData = [] 

437 # Resolve special combinations where the expected output is different 

438 for input, corrected in testLabels: 

439 for dataId in testIds: 

440 if input is None: 

441 if dataId["filter"] == "i2": 

442 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

443 else: 

444 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701")) 

445 elif input == afwImage.FilterLabel(band="i") and dataId["filter"] == "i2": 

446 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

447 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"): 

448 # Contradictory inputs, leave as-is 

449 data = (input, dataId, input) 

450 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2": 

451 # Contradictory inputs, leave as-is 

452 data = (input, dataId, input) 

453 else: 

454 data = (input, dataId, corrected) 

455 testData.append(data) 

456 

457 mapper = MinMapper2(root=ROOT) 

458 for label, dataId, corrected in testData: 

459 exposure = afwImage.ExposureF() 

460 exposure.setFilterLabel(label) 

461 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

462 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

463 

464 def testStandardizeFiltersFilterNoDefs(self): 

465 testLabels = [ 

466 None, 

467 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

468 afwImage.FilterLabel(band="i"), 

469 afwImage.FilterLabel(physical="i.MP9701"), 

470 afwImage.FilterLabel(band="i", physical="old-i"), 

471 afwImage.FilterLabel(physical="old-i"), 

472 afwImage.FilterLabel(physical="i2"), 

473 ] 

474 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

475 "i", "i.MP9701", "old-i", "i2", 

476 }] 

477 testData = [] 

478 # Resolve special combinations where the expected output is different 

479 for input in testLabels: 

480 for dataId in testIds: 

481 if input is None: 

482 # Can still get some filter info out of the Filter registry 

483 if dataId["filter"] == "i2": 

484 data = (input, dataId, 

485 afwImage.FilterLabel(band="i", physical="HSC-I2")) 

486 else: 

487 # Data ID maps to filter(s) with aliases; can't 

488 # unambiguously determine physical filter. 

489 data = (input, dataId, afwImage.FilterLabel(band="i")) 

490 else: 

491 data = (input, dataId, input) 

492 testData.append(data) 

493 

494 mapper = MinMapper1(root=ROOT) 

495 for label, dataId, corrected in testData: 

496 exposure = afwImage.ExposureF() 

497 exposure.setFilterLabel(label) 

498 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

499 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

500 

501 def testCalib(self): 

502 mapper = MinMapper2(root=ROOT) 

503 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True) 

504 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF") 

505 self.assertEqual(loc.getCppType(), "ExposureF") 

506 self.assertEqual(loc.getStorageName(), "FitsStorage") 

507 expectedRoot = ROOT 

508 expectedLocations = ["flat-05Am03-fi.fits"] 

509 self.assertEqual(loc.getStorage().root, expectedRoot) 

510 self.assertEqual(loc.getLocations(), expectedLocations) 

511 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

512 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650) 

513 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03") 

514 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i") 

515 checkCompression(self, loc.getAdditionalData()) 

516 

517 def testNames(self): 

518 self.assertEqual(MinMapper2.getCameraName(), "min") 

519 self.assertEqual(MinMapper2.getPackageName(), "moe") 

520 

521 @unittest.expectedFailure 

522 def testParentSearch(self): 

523 mapper = MinMapper2(root=ROOT) 

524 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

525 os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))) 

526 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))]) 

527 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

528 os.path.join(ROOT, 

529 os.path.join('testParentSearch', 'bar.fits[1]'))) 

530 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits[1]'))]) 

531 

532 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

533 os.path.join(ROOT, os.path.join('testParentSearch', 'baz.fits'))) 

534 self.assertEqual(paths, [os.path.join(ROOT, 

535 os.path.join('testParentSearch', '_parent', 'baz.fits'))]) 

536 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

537 os.path.join(ROOT, 

538 os.path.join('testParentSearch', 'baz.fits[1]'))) 

539 self.assertEqual(paths, [os.path.join(ROOT, 

540 os.path.join('testParentSearch', '_parent', 'baz.fits[1]'))]) 

541 

542 def testSkymapLookups(self): 

543 """Test that metadata lookups don't try to get skymap data ID values 

544 from the registry. 

545 """ 

546 mapper = MinMapper2(root=ROOT) 

547 butler = dafPersist.Butler(mapper=mapper) 

548 with self.assertRaises(RuntimeError) as manager: 

549 butler.dataRef("forced_src", visit=787650, ccd=13) 

550 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception)) 

551 # We're mostly concerned that the statements below will raise an 

552 # exception; if they don't, it's not likely the following tests will 

553 # fail. 

554 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0) 

555 self.assertEqual(len(subset), 1) 

556 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0) 

557 self.assertFalse(dataRef.datasetExists("forced_src")) 

558 

559 

560class Mapper3TestCase(unittest.TestCase): 

561 """A test case for a mapper subclass which does not assign packageName.""" 

562 

563 def testPackageName(self): 

564 with self.assertRaises(ValueError): 

565 MinMapper3() 

566 with self.assertRaises(ValueError): 

567 MinMapper3.getPackageName() 

568 

569 

570class ParentRegistryTestCase(unittest.TestCase): 

571 

572 @staticmethod 

573 def _createRegistry(path): 

574 cmd = """CREATE TABLE x( 

575 id INT, 

576 visit INT, 

577 filter TEXT, 

578 snap INT, 

579 raft TEXT, 

580 sensor TEXT, 

581 channel TEXT, 

582 taiObs TEXT, 

583 expTime REAL 

584 ); 

585 """ 

586 conn = sqlite3.connect(path) 

587 conn.cursor().execute(cmd) 

588 conn.commit() 

589 conn.close() 

590 

591 def setUp(self): 

592 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-") 

593 self.repoARoot = os.path.join(self.ROOT, 'a') 

594 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1) 

595 butler = dafPersist.Butler(outputs=args) 

596 self._createRegistry(os.path.join(self.repoARoot, 'registry.sqlite3')) 

597 del butler 

598 

599 def tearDown(self): 

600 # the butler sql registry closes its database connection in __del__. 

601 # To trigger __del__ we explicitly collect the garbage here. If we 

602 # find having or closing the open database connection is a problem in 

603 # production code, we may need to add api to butler to explicity 

604 # release database connections (and maybe other things like in-memory 

605 # cached objects). 

606 gc.collect() 

607 if os.path.exists(self.ROOT): 

608 shutil.rmtree(self.ROOT) 

609 

610 def test(self): 

611 """Verify that when the child repo does not have a registry it is 

612 assigned the registry from the parent. 

613 """ 

614 repoBRoot = os.path.join(self.ROOT, 'b') 

615 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

616 # This way of getting the registry from the mapping is obviously going 

617 # way into private members and the python lambda implementation code. 

618 # It is very brittle and should not be duplicated in user code 

619 # or any location that is not trivial to fix along with changes to the 

620 # CameraMapper or Mapping. 

621 registryA = butler._repos.inputs()[0].repo._mapper.registry 

622 registryB = butler._repos.outputs()[0].repo._mapper.registry 

623 self.assertEqual(id(registryA), id(registryB)) 

624 

625 self._createRegistry(os.path.join(repoBRoot, 'registry.sqlite3')) 

626 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

627 # see above; don't copy this way of getting the registry. 

628 registryA = butler._repos.inputs()[0].repo._mapper.registry 

629 registryB = butler._repos.outputs()[0].repo._mapper.registry 

630 self.assertNotEqual(id(registryA), id(registryB)) 

631 

632 

633class MissingPolicyKeyTestCase(unittest.TestCase): 

634 

635 def testGetRaises(self): 

636 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

637 # MinMapper1 does not specify a template for the raw dataset type so 

638 # trying to use it for get should raise 

639 with self.assertRaises(RuntimeError) as contextManager: 

640 butler.get('raw') 

641 # This test demonstrates and verifies that simple use of the incomplete 

642 # dataset type returns a helpful (I hope) error message. 

643 self.assertEqual( 

644 str(contextManager.exception), 

645 'Template is not defined for the raw dataset type, ' 

646 'it must be set before it can be used.') 

647 with self.assertRaises(RuntimeError) as contextManager: 

648 butler.queryMetadata('raw', 'unused', {}) 

649 

650 def testQueryMetadataRaises(self): 

651 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

652 # MinMapper1 does not specify a template for the raw dataset type so 

653 # trying to use it for queryMetadata should raise 

654 with self.assertRaises(RuntimeError) as contextManager: 

655 butler.queryMetadata('raw', 'unused', {}) 

656 # This test demonstrates and verifies that simple use of the incomplete 

657 # dataset type returns a helpful (I hope) error message. 

658 self.assertEqual( 

659 str(contextManager.exception), 

660 'Template is not defined for the raw dataset type, ' 

661 'it must be set before it can be used.') 

662 

663 def testFilenameRaises(self): 

664 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

665 # MinMapper1 does not specify a template for the raw dataset type so 

666 # trying to use it for <datasetType>_filename should raise 

667 with self.assertRaises(RuntimeError) as contextManager: 

668 butler.get('raw_filename') 

669 # This test demonstrates and verifies that simple use of the 

670 # incomplete dataset type returns a helpful (I hope) error message. 

671 self.assertEqual( 

672 str(contextManager.exception), 

673 'Template is not defined for the raw dataset type, ' 

674 'it must be set before it can be used.') 

675 

676 def testWcsRaises(self): 

677 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

678 # MinMapper1 does not specify a template for the raw dataset type so 

679 # trying to use it for <datasetType>_wcs should raise 

680 with self.assertRaises(RuntimeError) as contextManager: 

681 butler.get('raw_wcs') 

682 # This test demonstrates and verifies that simple use of the 

683 # incomplete dataset type returns a helpful (I hope) error message. 

684 self.assertEqual( 

685 str(contextManager.exception), 

686 'Template is not defined for the raw dataset type, ' 

687 'it must be set before it can be used.') 

688 

689 def testConflictRaises(self): 

690 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml")) 

691 with self.assertRaisesRegex( 

692 ValueError, 

693 r"Duplicate mapping policy for dataset type packages"): 

694 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841 

695 

696 

697class MemoryTester(lsst.utils.tests.MemoryTestCase): 

698 pass 

699 

700 

701if __name__ == '__main__': 701 ↛ 702line 701 didn't jump to line 702, because the condition on line 701 was never true

702 lsst.utils.tests.init() 

703 unittest.main()