Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import gc 

23import os 

24import sqlite3 

25import unittest 

26import tempfile 

27 

28import numpy as np 

29 

30import lsst.utils.tests 

31import lsst.geom as geom 

32import lsst.afw.image as afwImage 

33import lsst.afw.table as afwTable 

34import lsst.daf.persistence as dafPersist 

35import lsst.obs.base 

36import shutil 

37 

38from lsst.obs.base.test import BaseMapper 

39 

40ROOT = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43def setup_module(module): 

44 lsst.utils.tests.init() 

45 

46 

47class MinCam(lsst.obs.base.Instrument): 

48 @property 

49 def filterDefinitions(self): 

50 return lsst.obs.base.FilterDefinitionCollection( 

51 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374), 

52 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487), 

53 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"}, 

54 lambdaEff=628), 

55 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", alias={"old-i"}, 

56 lambdaEff=778), 

57 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170), 

58 # afw_name is so special-cased that only a real example will work 

59 lsst.obs.base.FilterDefinition(physical_filter="HSC-I2", band="i", afw_name="i2", lambdaEff=623), 

60 ) 

61 

62 @classmethod 

63 def getName(cls): 

64 return "min" 

65 

66 def getCamera(self): 

67 raise NotImplementedError() 

68 

69 def register(self, registry): 

70 raise NotImplementedError() 

71 

72 def getRawFormatter(self, dataId): 

73 raise NotImplementedError() 

74 

75 def makeDataIdTranslatorFactory(self): 

76 raise NotImplementedError() 

77 

78 

79class MinMapper1(lsst.obs.base.CameraMapper): 

80 packageName = 'larry' 

81 

82 def __init__(self, **kwargs): 

83 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

84 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

85 return 

86 

87 def std_x(self, item, dataId): 

88 return float(item) 

89 

90 @classmethod 

91 def getCameraName(cls): 

92 """Return the name of the camera that this CameraMapper is for.""" 

93 return "min" 

94 

95 @classmethod 

96 def getPackageDir(cls): 

97 return "/path/to/nowhere" 

98 

99 

100class MinMapper2(lsst.obs.base.CameraMapper): 

101 packageName = 'moe' 

102 _gen3instrument = MinCam 

103 

104 # CalibRoot in policy 

105 # needCalibRegistry 

106 def __init__(self, **kwargs): 

107 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml")) 

108 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, 

109 registry="cfhtls.sqlite3", **kwargs) 

110 return 

111 

112 def _transformId(self, dataId): 

113 return dataId 

114 

115 def _extractDetectorName(self, dataId): 

116 return "ccd00" 

117 

118 def std_x(self, item, dataId): 

119 return float(item) 

120 

121 @classmethod 

122 def getCameraName(cls): 

123 """Return the name of the camera that this CameraMapper is for.""" 

124 return "min" 

125 

126 @classmethod 

127 def getPackageDir(cls): 

128 return "/path/to/nowhere" 

129 

130 

131# does not assign packageName 

132class MinMapper3(lsst.obs.base.CameraMapper): 

133 

134 def __init__(self, **kwargs): 

135 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

136 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT) 

137 return 

138 

139 @classmethod 

140 def getPackageDir(cls): 

141 return "/path/to/nowhere" 

142 

143 

144def checkCompression(testCase, additionalData): 

145 """Check that compression settings are present 

146 

147 We check that we can access the required settings, and that 

148 the seed is non-zero (zero causes lsst.afw.math.Random to fail). 

149 """ 

150 for plane in ("image", "mask", "variance"): 

151 for entry in ("compression.algorithm", 

152 "compression.columns", 

153 "compression.rows", 

154 "compression.quantizeLevel", 

155 "scaling.algorithm", 

156 "scaling.bitpix", 

157 "scaling.maskPlanes", 

158 "scaling.seed", 

159 "scaling.quantizeLevel", 

160 "scaling.quantizePad", 

161 "scaling.fuzz", 

162 "scaling.bscale", 

163 "scaling.bzero", 

164 ): 

165 additionalData.getScalar(plane + "." + entry) 

166 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0) 

167 

168 

169class Mapper1TestCase(unittest.TestCase): 

170 """A test case for the mapper used by the data butler.""" 

171 

172 def setUp(self): 

173 self.mapper = MinMapper1(root=ROOT) 

174 

175 def tearDown(self): 

176 del self.mapper 

177 

178 def testGetDatasetTypes(self): 

179 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

180 # Add the expected additional types to what the base class provides 

181 expectedTypes.extend(["x", "x_filename", 

182 "badSourceHist", "badSourceHist_filename", ]) 

183 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes)) 

184 

185 def testMap(self): 

186 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True) 

187 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

188 self.assertEqual(loc.getCppType(), "BoxI") 

189 self.assertEqual(loc.getStorageName(), "PickleStorage") 

190 expectedRoot = ROOT 

191 expectedLocations = ["foo-1,1.pickle"] 

192 self.assertEqual(loc.getStorage().root, expectedRoot) 

193 self.assertEqual(loc.getLocations(), expectedLocations) 

194 self.assertEqual(loc.getAdditionalData().toString(), 

195 "sensor = \"1,1\"\n") 

196 

197 def testQueryMetadata(self): 

198 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)]) 

199 

200 def testStandardize(self): 

201 self.assertTrue(self.mapper.canStandardize("x")) 

202 self.assertFalse(self.mapper.canStandardize("badSourceHist")) 

203 self.assertFalse(self.mapper.canStandardize("notPresent")) 

204 result = self.mapper.standardize("x", 3, None) 

205 self.assertIsInstance(result, float) 

206 self.assertEqual(result, 3.0) 

207 result = self.mapper.standardize("x", 3.14, None) 

208 self.assertIsInstance(result, float) 

209 self.assertEqual(result, 3.14) 

210 result = self.mapper.standardize("x", "3.14", None) 

211 self.assertIsInstance(result, float) 

212 self.assertEqual(result, 3.14) 

213 

214 def testNames(self): 

215 self.assertEqual(MinMapper1.getCameraName(), "min") 

216 self.assertEqual(MinMapper1.getPackageName(), "larry") 

217 

218 

219class Mapper2TestCase(unittest.TestCase): 

220 """A test case for the mapper used by the data butler.""" 

221 

222 def setUp(self): 

223 super().setUp() 

224 # Force a standard set of filters even for tests that don't use 

225 # MinCam directly. 

226 MinCam() 

227 

228 def testGetDatasetTypes(self): 

229 mapper = MinMapper2(root=ROOT) 

230 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

231 # Add the expected additional types to what the base class provides 

232 expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub", 

233 "raw", "raw_md", "raw_filename", "raw_sub", 

234 "some", "some_filename", "some_md", "some_sub", 

235 "someCatalog", "someCatalog_md", "someCatalog_filename", 

236 "someCatalog_len", "someCatalog_schema", 

237 "forced_src", "forced_src_md", "forced_src_filename", 

238 "forced_src_len", "forced_src_schema", 

239 "other_sub", "other_filename", "other_md", "other", 

240 "someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md", 

241 "someFz_sub", "someFz_md", "someGz_sub", 

242 "someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox", 

243 "someExp", "someExp_filename", "someExp_md", "someExp_sub", 

244 "someExp_bbox", "someExp_filterLabel", "someExp_photoCalib", 

245 "someExp_visitInfo", "someExp_detector", "someExp_filter", 

246 "someExp_header_wcs", "someExp_wcs", 

247 ]) 

248 self.assertEqual(set(mapper.getDatasetTypes()), 

249 set(expectedTypes)) 

250 

251 def testMap(self): 

252 mapper = MinMapper2(root=ROOT) 

253 loc = mapper.map("raw", {"ccd": 13}, write=True) 

254 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

255 self.assertEqual(loc.getCppType(), "ImageU") 

256 self.assertEqual(loc.getStorageName(), "FitsStorage") 

257 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

258 self.assertEqual(loc.getStorage().root, ROOT) 

259 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

260 checkCompression(self, loc.getAdditionalData()) 

261 

262 def testSubMap(self): 

263 bbox = geom.BoxI(geom.Point2I(200, 100), 

264 geom.Extent2I(300, 400)) 

265 mapper = MinMapper2(root=ROOT) 

266 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True) 

267 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

268 self.assertEqual(loc.getCppType(), "ImageU") 

269 self.assertEqual(loc.getStorageName(), "FitsStorage") 

270 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

271 self.assertEqual(loc.getStorage().root, ROOT) 

272 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

273 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

274 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

275 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

276 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

277 checkCompression(self, loc.getAdditionalData()) 

278 

279 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True) 

280 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

281 self.assertEqual(loc.getCppType(), "ImageU") 

282 self.assertEqual(loc.getStorageName(), "FitsStorage") 

283 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

284 self.assertEqual(loc.getStorage().root, ROOT) 

285 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

286 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

287 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

288 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

289 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

290 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT") 

291 checkCompression(self, loc.getAdditionalData()) 

292 

293 def testCatalogExtras(self): 

294 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) 

295 schema = afwTable.Schema() 

296 aa = schema.addField("a", type=np.int32, doc="a") 

297 bb = schema.addField("b", type=np.float64, doc="b") 

298 catalog = lsst.afw.table.BaseCatalog(schema) 

299 row = catalog.addNew() 

300 row.set(aa, 12345) 

301 row.set(bb, 1.2345) 

302 size = len(catalog) 

303 dataId = dict(visit=123, ccd=45) 

304 butler.put(catalog, "someCatalog", dataId) 

305 filename = butler.get("someCatalog_filename", dataId)[0] 

306 try: 

307 self.assertTrue(os.path.exists(filename)) 

308 self.assertEqual(butler.get("someCatalog_schema", dataId), schema) 

309 self.assertEqual(butler.get("someCatalog_len", dataId), size) 

310 header = butler.get("someCatalog_md", dataId) 

311 self.assertEqual(header.getScalar("NAXIS2"), size) 

312 finally: 

313 try: 

314 os.remove(filename) 

315 except OSError as exc: 

316 print("Warning: could not remove file %r: %s" % (filename, exc)) 

317 

318 def testImage(self): 

319 mapper = MinMapper2(root=ROOT) 

320 loc = mapper.map("some", dict(ccd=35)) 

321 expectedLocations = ["bar-35.fits"] 

322 self.assertEqual(loc.getStorage().root, ROOT) 

323 self.assertEqual(loc.getLocations(), expectedLocations) 

324 

325 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

326 image = butler.get("some", ccd=35) 

327 self.assertEqual(image.getFilter().getName(), "r") 

328 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

329 

330 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox()) 

331 

332 bbox = geom.BoxI(geom.Point2I(200, 100), 

333 geom.Extent2I(300, 400)) 

334 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

335 self.assertEqual(image.getHeight(), 400) 

336 self.assertEqual(image.getWidth(), 300) 

337 

338 def testFilter(self): 

339 """Test that the same (patched) filter is returned through all Butler 

340 retrieval paths. 

341 """ 

342 mapper = MinMapper2(root=ROOT) 

343 

344 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

345 image = butler.get("someExp", ccd=35) 

346 filter = butler.get("someExp_filterLabel", ccd=35) 

347 # Test only valid with a complete filter 

348 self.assertEqual(image.getFilterLabel(), afwImage.FilterLabel(band="r", physical="r.MP9601")) 

349 # Datasets should give consistent answers 

350 self.assertEqual(filter, image.getFilterLabel()) 

351 

352 def testDetector(self): 

353 mapper = MinMapper2(root=ROOT) 

354 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

355 detector = butler.get("raw_detector", ccd=0) 

356 self.assertEqual(detector.getName(), "ccd00") 

357 

358 def testGzImage(self): 

359 mapper = MinMapper2(root=ROOT) 

360 loc = mapper.map("someGz", dict(ccd=35)) 

361 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")] 

362 self.assertEqual(loc.getStorage().root, ROOT) 

363 self.assertEqual(loc.getLocations(), expectedLocations) 

364 

365 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

366 image = butler.get("someGz", ccd=35) 

367 self.assertEqual(image.getFilter().getName(), "r") 

368 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

369 

370 bbox = geom.BoxI(geom.Point2I(200, 100), 

371 geom.Extent2I(300, 400)) 

372 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

373 self.assertEqual(image.getHeight(), 400) 

374 self.assertEqual(image.getWidth(), 300) 

375 

376 def testFzImage(self): 

377 mapper = MinMapper2(root=ROOT) 

378 loc = mapper.map("someFz", dict(ccd=35)) 

379 expectedRoot = ROOT 

380 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")] 

381 self.assertEqual(loc.getStorage().root, expectedRoot) 

382 self.assertEqual(loc.getLocations(), expectedLocations) 

383 

384 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

385 image = butler.get("someFz", ccd=35) 

386 self.assertEqual(image.getFilter().getName(), "r") 

387 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

388 

389 bbox = geom.BoxI(geom.Point2I(200, 100), 

390 geom.Extent2I(300, 400)) 

391 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

392 self.assertEqual(image.getHeight(), 400) 

393 self.assertEqual(image.getWidth(), 300) 

394 

395 def testButlerQueryMetadata(self): 

396 mapper = MinMapper2(root=ROOT) 

397 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

398 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, 

399 "taiObs": "2005-04-02T09:24:49.933440000"} 

400 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731]) 

401 self.assertEqual(butler.queryMetadata("other", "visit", 

402 visit=kwargs["visit"], ccd=kwargs["ccd"], 

403 taiObs=kwargs["taiObs"], filter=kwargs["filter"]), 

404 [787731]) 

405 # now test we get no matches if ccd is out of range 

406 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), []) 

407 

408 def testQueryMetadata(self): 

409 mapper = MinMapper2(root=ROOT) 

410 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), 

411 [(x,) for x in range(36) if x != 3]) 

412 

413 def testStandardize(self): 

414 mapper = MinMapper2(root=ROOT) 

415 self.assertEqual(mapper.canStandardize("raw"), True) 

416 self.assertEqual(mapper.canStandardize("notPresent"), False) 

417 

418 def testStandardizeFiltersFilterDefs(self): 

419 testLabels = [ 

420 (None, None), 

421 (afwImage.FilterLabel(band="i", physical="i.MP9701"), 

422 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

423 (afwImage.FilterLabel(band="i"), afwImage.FilterLabel(band="r", physical="i.MP9701")), 

424 (afwImage.FilterLabel(physical="i.MP9701"), 

425 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

426 (afwImage.FilterLabel(band="i", physical="old-i"), 

427 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

428 (afwImage.FilterLabel(physical="old-i"), 

429 afwImage.FilterLabel(band="i", physical="i.MP9701")), 

430 (afwImage.FilterLabel(physical="i2"), afwImage.FilterLabel(band="i", physical="HSC-I2")), 

431 ] 

432 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

433 "i", "i.MP9701", "old-i", "i2", 

434 }] 

435 testData = [] 

436 # Resolve special combinations where the expected output is different 

437 for input, corrected in testLabels: 

438 for dataId in testIds: 

439 if input is None: 

440 if dataId["filter"] == "i": 

441 data = (input, dataId, afwImage.FilterLabel(band="i")) 

442 elif dataId["filter"] == "i2": 

443 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

444 else: 

445 data = (input, dataId, afwImage.FilterLabel(band="i", physical="i.MP9701")) 

446 elif input == afwImage.FilterLabel(band="i"): 

447 if dataId["filter"] == "i": 

448 # There are two "i" filters, can't tell which 

449 data = (input, dataId, input) 

450 elif dataId["filter"] == "i2": 

451 data = (input, dataId, afwImage.FilterLabel(band="i", physical="HSC-I2")) 

452 elif corrected.physicalLabel == "HSC-I2" and dataId["filter"] in ("i.MP9701", "old-i"): 

453 # Contradictory inputs, leave as-is 

454 data = (input, dataId, input) 

455 elif corrected.physicalLabel == "i.MP9701" and dataId["filter"] == "i2": 

456 # Contradictory inputs, leave as-is 

457 data = (input, dataId, input) 

458 else: 

459 data = (input, dataId, corrected) 

460 testData.append(data) 

461 

462 mapper = MinMapper2(root=ROOT) 

463 for label, dataId, corrected in testData: 

464 exposure = afwImage.ExposureF() 

465 exposure.setFilterLabel(label) 

466 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

467 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

468 

469 def testStandardizeFiltersFilterNoDefs(self): 

470 testLabels = [ 

471 None, 

472 afwImage.FilterLabel(band="i", physical="i.MP9701"), 

473 afwImage.FilterLabel(band="i"), 

474 afwImage.FilterLabel(physical="i.MP9701"), 

475 afwImage.FilterLabel(band="i", physical="old-i"), 

476 afwImage.FilterLabel(physical="old-i"), 

477 afwImage.FilterLabel(physical="i2"), 

478 ] 

479 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

480 "i", "i.MP9701", "old-i", "i2", 

481 }] 

482 testData = [] 

483 # Resolve special combinations where the expected output is different 

484 for input in testLabels: 

485 for dataId in testIds: 

486 if input is None: 

487 # Can still get some filter info out of the Filter registry 

488 if dataId["filter"] == "i2": 

489 data = (input, dataId, 

490 afwImage.FilterLabel(band="i", physical="HSC-I2")) 

491 else: 

492 # Data ID maps to filter(s) with aliases; can't 

493 # unambiguously determine physical filter. 

494 data = (input, dataId, afwImage.FilterLabel(band="i")) 

495 else: 

496 data = (input, dataId, input) 

497 testData.append(data) 

498 

499 mapper = MinMapper1(root=ROOT) 

500 for label, dataId, corrected in testData: 

501 exposure = afwImage.ExposureF() 

502 exposure.setFilterLabel(label) 

503 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

504 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

505 

506 def testCalib(self): 

507 mapper = MinMapper2(root=ROOT) 

508 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True) 

509 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF") 

510 self.assertEqual(loc.getCppType(), "ExposureF") 

511 self.assertEqual(loc.getStorageName(), "FitsStorage") 

512 expectedRoot = ROOT 

513 expectedLocations = ["flat-05Am03-fi.fits"] 

514 self.assertEqual(loc.getStorage().root, expectedRoot) 

515 self.assertEqual(loc.getLocations(), expectedLocations) 

516 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

517 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650) 

518 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03") 

519 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i") 

520 checkCompression(self, loc.getAdditionalData()) 

521 

522 def testNames(self): 

523 self.assertEqual(MinMapper2.getCameraName(), "min") 

524 self.assertEqual(MinMapper2.getPackageName(), "moe") 

525 

526 @unittest.expectedFailure 

527 def testParentSearch(self): 

528 mapper = MinMapper2(root=ROOT) 

529 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

530 os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))) 

531 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))]) 

532 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

533 os.path.join(ROOT, 

534 os.path.join('testParentSearch', 'bar.fits[1]'))) 

535 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits[1]'))]) 

536 

537 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

538 os.path.join(ROOT, os.path.join('testParentSearch', 'baz.fits'))) 

539 self.assertEqual(paths, [os.path.join(ROOT, 

540 os.path.join('testParentSearch', '_parent', 'baz.fits'))]) 

541 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

542 os.path.join(ROOT, 

543 os.path.join('testParentSearch', 'baz.fits[1]'))) 

544 self.assertEqual(paths, [os.path.join(ROOT, 

545 os.path.join('testParentSearch', '_parent', 'baz.fits[1]'))]) 

546 

547 def testSkymapLookups(self): 

548 """Test that metadata lookups don't try to get skymap data ID values 

549 from the registry. 

550 """ 

551 mapper = MinMapper2(root=ROOT) 

552 butler = dafPersist.Butler(mapper=mapper) 

553 with self.assertRaises(RuntimeError) as manager: 

554 butler.dataRef("forced_src", visit=787650, ccd=13) 

555 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception)) 

556 # We're mostly concerned that the statements below will raise an 

557 # exception; if they don't, it's not likely the following tests will 

558 # fail. 

559 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0) 

560 self.assertEqual(len(subset), 1) 

561 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0) 

562 self.assertFalse(dataRef.datasetExists("forced_src")) 

563 

564 

565class Mapper3TestCase(unittest.TestCase): 

566 """A test case for a mapper subclass which does not assign packageName.""" 

567 

568 def testPackageName(self): 

569 with self.assertRaises(ValueError): 

570 MinMapper3() 

571 with self.assertRaises(ValueError): 

572 MinMapper3.getPackageName() 

573 

574 

575class ParentRegistryTestCase(unittest.TestCase): 

576 

577 @staticmethod 

578 def _createRegistry(path): 

579 cmd = """CREATE TABLE x( 

580 id INT, 

581 visit INT, 

582 filter TEXT, 

583 snap INT, 

584 raft TEXT, 

585 sensor TEXT, 

586 channel TEXT, 

587 taiObs TEXT, 

588 expTime REAL 

589 ); 

590 """ 

591 conn = sqlite3.connect(path) 

592 conn.cursor().execute(cmd) 

593 conn.commit() 

594 conn.close() 

595 

596 def setUp(self): 

597 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-") 

598 self.repoARoot = os.path.join(self.ROOT, 'a') 

599 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1) 

600 butler = dafPersist.Butler(outputs=args) 

601 self._createRegistry(os.path.join(self.repoARoot, 'registry.sqlite3')) 

602 del butler 

603 

604 def tearDown(self): 

605 # the butler sql registry closes its database connection in __del__. 

606 # To trigger __del__ we explicitly collect the garbage here. If we 

607 # find having or closing the open database connection is a problem in 

608 # production code, we may need to add api to butler to explicity 

609 # release database connections (and maybe other things like in-memory 

610 # cached objects). 

611 gc.collect() 

612 if os.path.exists(self.ROOT): 

613 shutil.rmtree(self.ROOT) 

614 

615 def test(self): 

616 """Verify that when the child repo does not have a registry it is 

617 assigned the registry from the parent. 

618 """ 

619 repoBRoot = os.path.join(self.ROOT, 'b') 

620 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

621 # This way of getting the registry from the mapping is obviously going 

622 # way into private members and the python lambda implementation code. 

623 # It is very brittle and should not be duplicated in user code 

624 # or any location that is not trivial to fix along with changes to the 

625 # CameraMapper or Mapping. 

626 registryA = butler._repos.inputs()[0].repo._mapper.registry 

627 registryB = butler._repos.outputs()[0].repo._mapper.registry 

628 self.assertEqual(id(registryA), id(registryB)) 

629 

630 self._createRegistry(os.path.join(repoBRoot, 'registry.sqlite3')) 

631 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

632 # see above; don't copy this way of getting the registry. 

633 registryA = butler._repos.inputs()[0].repo._mapper.registry 

634 registryB = butler._repos.outputs()[0].repo._mapper.registry 

635 self.assertNotEqual(id(registryA), id(registryB)) 

636 

637 

638class MissingPolicyKeyTestCase(unittest.TestCase): 

639 

640 def testGetRaises(self): 

641 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

642 # MinMapper1 does not specify a template for the raw dataset type so 

643 # trying to use it for get should raise 

644 with self.assertRaises(RuntimeError) as contextManager: 

645 butler.get('raw') 

646 # This test demonstrates and verifies that simple use of the incomplete 

647 # dataset type returns a helpful (I hope) error message. 

648 self.assertEqual( 

649 str(contextManager.exception), 

650 'Template is not defined for the raw dataset type, ' 

651 'it must be set before it can be used.') 

652 with self.assertRaises(RuntimeError) as contextManager: 

653 butler.queryMetadata('raw', 'unused', {}) 

654 

655 def testQueryMetadataRaises(self): 

656 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

657 # MinMapper1 does not specify a template for the raw dataset type so 

658 # trying to use it for queryMetadata should raise 

659 with self.assertRaises(RuntimeError) as contextManager: 

660 butler.queryMetadata('raw', 'unused', {}) 

661 # This test demonstrates and verifies that simple use of the incomplete 

662 # dataset type returns a helpful (I hope) error message. 

663 self.assertEqual( 

664 str(contextManager.exception), 

665 'Template is not defined for the raw dataset type, ' 

666 'it must be set before it can be used.') 

667 

668 def testFilenameRaises(self): 

669 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

670 # MinMapper1 does not specify a template for the raw dataset type so 

671 # trying to use it for <datasetType>_filename should raise 

672 with self.assertRaises(RuntimeError) as contextManager: 

673 butler.get('raw_filename') 

674 # This test demonstrates and verifies that simple use of the 

675 # incomplete dataset type returns a helpful (I hope) error message. 

676 self.assertEqual( 

677 str(contextManager.exception), 

678 'Template is not defined for the raw dataset type, ' 

679 'it must be set before it can be used.') 

680 

681 def testWcsRaises(self): 

682 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

683 # MinMapper1 does not specify a template for the raw dataset type so 

684 # trying to use it for <datasetType>_wcs should raise 

685 with self.assertRaises(RuntimeError) as contextManager: 

686 butler.get('raw_wcs') 

687 # This test demonstrates and verifies that simple use of the 

688 # incomplete dataset type returns a helpful (I hope) error message. 

689 self.assertEqual( 

690 str(contextManager.exception), 

691 'Template is not defined for the raw dataset type, ' 

692 'it must be set before it can be used.') 

693 

694 def testConflictRaises(self): 

695 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml")) 

696 with self.assertRaisesRegex( 

697 ValueError, 

698 r"Duplicate mapping policy for dataset type packages"): 

699 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841 

700 

701 

702class MemoryTester(lsst.utils.tests.MemoryTestCase): 

703 pass 

704 

705 

706if __name__ == '__main__': 706 ↛ 707line 706 didn't jump to line 707, because the condition on line 706 was never true

707 lsst.utils.tests.init() 

708 unittest.main()