Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of obs_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import gc 

23import os 

24import sqlite3 

25import unittest 

26import tempfile 

27 

28import numpy as np 

29 

30import lsst.utils.tests 

31import lsst.geom as geom 

32import lsst.afw.image as afwImage 

33import lsst.afw.table as afwTable 

34import lsst.daf.persistence as dafPersist 

35import lsst.obs.base 

36import shutil 

37 

38from lsst.obs.base.test import BaseMapper 

39 

40ROOT = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43def setup_module(module): 

44 lsst.utils.tests.init() 

45 

46 

47class MinCam(lsst.obs.base.Instrument): 

48 @property 

49 def filterDefinitions(self): 

50 return lsst.obs.base.FilterDefinitionCollection( 

51 lsst.obs.base.FilterDefinition(physical_filter="u.MP9301", band="u", lambdaEff=374), 

52 lsst.obs.base.FilterDefinition(physical_filter="g.MP9401", band="g", lambdaEff=487), 

53 lsst.obs.base.FilterDefinition(physical_filter="r.MP9601", band="r", alias={"old-r"}, 

54 lambdaEff=628), 

55 lsst.obs.base.FilterDefinition(physical_filter="i.MP9701", band="i", lambdaEff=778), 

56 lsst.obs.base.FilterDefinition(physical_filter="z.MP9801", band="z", lambdaEff=1170), 

57 # afw_name is so special-cased that only a real example will work 

58 lsst.obs.base.FilterDefinition(physical_filter="HSC-R2", band="r", afw_name="r2", lambdaEff=623), 

59 ) 

60 

61 @classmethod 

62 def getName(cls): 

63 return "min" 

64 

65 def getCamera(self): 

66 raise NotImplementedError() 

67 

68 def register(self, registry): 

69 raise NotImplementedError() 

70 

71 def getRawFormatter(self, dataId): 

72 raise NotImplementedError() 

73 

74 def makeDataIdTranslatorFactory(self): 

75 raise NotImplementedError() 

76 

77 

78class MinMapper1(lsst.obs.base.CameraMapper): 

79 packageName = 'larry' 

80 

81 def __init__(self, **kwargs): 

82 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

83 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, **kwargs) 

84 return 

85 

86 def std_x(self, item, dataId): 

87 return float(item) 

88 

89 @classmethod 

90 def getCameraName(cls): 

91 """Return the name of the camera that this CameraMapper is for.""" 

92 return "min" 

93 

94 @classmethod 

95 def getPackageDir(cls): 

96 return "/path/to/nowhere" 

97 

98 

99class MinMapper2(lsst.obs.base.CameraMapper): 

100 packageName = 'moe' 

101 _gen3instrument = MinCam 

102 

103 # CalibRoot in policy 

104 # needCalibRegistry 

105 def __init__(self, **kwargs): 

106 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper2.yaml")) 

107 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, 

108 registry="cfhtls.sqlite3", **kwargs) 

109 return 

110 

111 def _transformId(self, dataId): 

112 return dataId 

113 

114 def _extractDetectorName(self, dataId): 

115 return "ccd00" 

116 

117 def std_x(self, item, dataId): 

118 return float(item) 

119 

120 @classmethod 

121 def getCameraName(cls): 

122 """Return the name of the camera that this CameraMapper is for.""" 

123 return "min" 

124 

125 @classmethod 

126 def getPackageDir(cls): 

127 return "/path/to/nowhere" 

128 

129 

130# does not assign packageName 

131class MinMapper3(lsst.obs.base.CameraMapper): 

132 

133 def __init__(self, **kwargs): 

134 policy = dafPersist.Policy(os.path.join(ROOT, "MinMapper1.yaml")) 

135 lsst.obs.base.CameraMapper.__init__(self, policy=policy, repositoryDir=ROOT, root=ROOT) 

136 return 

137 

138 @classmethod 

139 def getPackageDir(cls): 

140 return "/path/to/nowhere" 

141 

142 

143def checkCompression(testCase, additionalData): 

144 """Check that compression settings are present 

145 

146 We check that we can access the required settings, and that 

147 the seed is non-zero (zero causes lsst.afw.math.Random to fail). 

148 """ 

149 for plane in ("image", "mask", "variance"): 

150 for entry in ("compression.algorithm", 

151 "compression.columns", 

152 "compression.rows", 

153 "compression.quantizeLevel", 

154 "scaling.algorithm", 

155 "scaling.bitpix", 

156 "scaling.maskPlanes", 

157 "scaling.seed", 

158 "scaling.quantizeLevel", 

159 "scaling.quantizePad", 

160 "scaling.fuzz", 

161 "scaling.bscale", 

162 "scaling.bzero", 

163 ): 

164 additionalData.getScalar(plane + "." + entry) 

165 testCase.assertNotEqual(additionalData.getScalar(plane + ".scaling.seed"), 0) 

166 

167 

168class Mapper1TestCase(unittest.TestCase): 

169 """A test case for the mapper used by the data butler.""" 

170 

171 def setUp(self): 

172 self.mapper = MinMapper1(root=ROOT) 

173 

174 def tearDown(self): 

175 del self.mapper 

176 

177 def testGetDatasetTypes(self): 

178 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

179 # Add the expected additional types to what the base class provides 

180 expectedTypes.extend(["x", "x_filename", 

181 "badSourceHist", "badSourceHist_filename", ]) 

182 self.assertEqual(set(self.mapper.getDatasetTypes()), set(expectedTypes)) 

183 

184 def testMap(self): 

185 loc = self.mapper.map("x", {"sensor": "1,1"}, write=True) 

186 self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI") 

187 self.assertEqual(loc.getCppType(), "BoxI") 

188 self.assertEqual(loc.getStorageName(), "PickleStorage") 

189 expectedRoot = ROOT 

190 expectedLocations = ["foo-1,1.pickle"] 

191 self.assertEqual(loc.getStorage().root, expectedRoot) 

192 self.assertEqual(loc.getLocations(), expectedLocations) 

193 self.assertEqual(loc.getAdditionalData().toString(), 

194 "sensor = \"1,1\"\n") 

195 

196 def testQueryMetadata(self): 

197 self.assertEqual(self.mapper.queryMetadata("x", ["sensor"], None), [("1,1",)]) 

198 

199 def testStandardize(self): 

200 self.assertTrue(self.mapper.canStandardize("x")) 

201 self.assertFalse(self.mapper.canStandardize("badSourceHist")) 

202 self.assertFalse(self.mapper.canStandardize("notPresent")) 

203 result = self.mapper.standardize("x", 3, None) 

204 self.assertIsInstance(result, float) 

205 self.assertEqual(result, 3.0) 

206 result = self.mapper.standardize("x", 3.14, None) 

207 self.assertIsInstance(result, float) 

208 self.assertEqual(result, 3.14) 

209 result = self.mapper.standardize("x", "3.14", None) 

210 self.assertIsInstance(result, float) 

211 self.assertEqual(result, 3.14) 

212 

213 def testNames(self): 

214 self.assertEqual(MinMapper1.getCameraName(), "min") 

215 self.assertEqual(MinMapper1.getPackageName(), "larry") 

216 

217 

218class Mapper2TestCase(unittest.TestCase): 

219 """A test case for the mapper used by the data butler.""" 

220 

221 def testGetDatasetTypes(self): 

222 mapper = MinMapper2(root=ROOT) 

223 expectedTypes = BaseMapper(ROOT).getDatasetTypes() 

224 # Add the expected additional types to what the base class provides 

225 expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub", 

226 "raw", "raw_md", "raw_filename", "raw_sub", 

227 "some", "some_filename", "some_md", "some_sub", 

228 "someCatalog", "someCatalog_md", "someCatalog_filename", 

229 "someCatalog_len", "someCatalog_schema", 

230 "forced_src", "forced_src_md", "forced_src_filename", 

231 "forced_src_len", "forced_src_schema", 

232 "other_sub", "other_filename", "other_md", "other", 

233 "someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md", 

234 "someFz_sub", "someFz_md", "someGz_sub", 

235 "someGz_bbox", "someFz_bbox", "some_bbox", "other_bbox", 

236 ]) 

237 self.assertEqual(set(mapper.getDatasetTypes()), 

238 set(expectedTypes)) 

239 

240 def testMap(self): 

241 mapper = MinMapper2(root=ROOT) 

242 loc = mapper.map("raw", {"ccd": 13}, write=True) 

243 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

244 self.assertEqual(loc.getCppType(), "ImageU") 

245 self.assertEqual(loc.getStorageName(), "FitsStorage") 

246 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

247 self.assertEqual(loc.getStorage().root, ROOT) 

248 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

249 checkCompression(self, loc.getAdditionalData()) 

250 

251 def testSubMap(self): 

252 bbox = geom.BoxI(geom.Point2I(200, 100), 

253 geom.Extent2I(300, 400)) 

254 mapper = MinMapper2(root=ROOT) 

255 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True) 

256 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

257 self.assertEqual(loc.getCppType(), "ImageU") 

258 self.assertEqual(loc.getStorageName(), "FitsStorage") 

259 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

260 self.assertEqual(loc.getStorage().root, ROOT) 

261 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

262 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

263 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

264 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

265 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

266 checkCompression(self, loc.getAdditionalData()) 

267 

268 loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox, "imageOrigin": "PARENT"}, write=True) 

269 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU") 

270 self.assertEqual(loc.getCppType(), "ImageU") 

271 self.assertEqual(loc.getStorageName(), "FitsStorage") 

272 self.assertEqual(loc.getLocations(), ["foo-13.fits"]) 

273 self.assertEqual(loc.getStorage().root, ROOT) 

274 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

275 self.assertEqual(loc.getAdditionalData().getScalar("width"), 300) 

276 self.assertEqual(loc.getAdditionalData().getScalar("height"), 400) 

277 self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200) 

278 self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100) 

279 self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"), "PARENT") 

280 checkCompression(self, loc.getAdditionalData()) 

281 

282 def testCatalogExtras(self): 

283 butler = dafPersist.Butler(root=ROOT, mapper=MinMapper2) 

284 schema = afwTable.Schema() 

285 aa = schema.addField("a", type=np.int32, doc="a") 

286 bb = schema.addField("b", type=np.float64, doc="b") 

287 catalog = lsst.afw.table.BaseCatalog(schema) 

288 row = catalog.addNew() 

289 row.set(aa, 12345) 

290 row.set(bb, 1.2345) 

291 size = len(catalog) 

292 dataId = dict(visit=123, ccd=45) 

293 butler.put(catalog, "someCatalog", dataId) 

294 filename = butler.get("someCatalog_filename", dataId)[0] 

295 try: 

296 self.assertTrue(os.path.exists(filename)) 

297 self.assertEqual(butler.get("someCatalog_schema", dataId), schema) 

298 self.assertEqual(butler.get("someCatalog_len", dataId), size) 

299 header = butler.get("someCatalog_md", dataId) 

300 self.assertEqual(header.getScalar("NAXIS2"), size) 

301 finally: 

302 try: 

303 os.remove(filename) 

304 except OSError as exc: 

305 print("Warning: could not remove file %r: %s" % (filename, exc)) 

306 

307 def testImage(self): 

308 mapper = MinMapper2(root=ROOT) 

309 loc = mapper.map("some", dict(ccd=35)) 

310 expectedLocations = ["bar-35.fits"] 

311 self.assertEqual(loc.getStorage().root, ROOT) 

312 self.assertEqual(loc.getLocations(), expectedLocations) 

313 

314 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

315 image = butler.get("some", ccd=35) 

316 self.assertEqual(image.getFilter().getName(), "r") 

317 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

318 

319 self.assertEqual(butler.get("some_bbox", ccd=35), image.getBBox()) 

320 

321 bbox = geom.BoxI(geom.Point2I(200, 100), 

322 geom.Extent2I(300, 400)) 

323 image = butler.get("some_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

324 self.assertEqual(image.getHeight(), 400) 

325 self.assertEqual(image.getWidth(), 300) 

326 

327 def testDetector(self): 

328 mapper = MinMapper2(root=ROOT) 

329 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

330 detector = butler.get("raw_detector", ccd=0) 

331 self.assertEqual(detector.getName(), "ccd00") 

332 

333 def testGzImage(self): 

334 mapper = MinMapper2(root=ROOT) 

335 loc = mapper.map("someGz", dict(ccd=35)) 

336 expectedLocations = [os.path.join("gz", "bar-35.fits.gz")] 

337 self.assertEqual(loc.getStorage().root, ROOT) 

338 self.assertEqual(loc.getLocations(), expectedLocations) 

339 

340 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

341 image = butler.get("someGz", ccd=35) 

342 self.assertEqual(image.getFilter().getName(), "r") 

343 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

344 

345 bbox = geom.BoxI(geom.Point2I(200, 100), 

346 geom.Extent2I(300, 400)) 

347 image = butler.get("someGz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

348 self.assertEqual(image.getHeight(), 400) 

349 self.assertEqual(image.getWidth(), 300) 

350 

351 def testFzImage(self): 

352 mapper = MinMapper2(root=ROOT) 

353 loc = mapper.map("someFz", dict(ccd=35)) 

354 expectedRoot = ROOT 

355 expectedLocations = [os.path.join("fz", "bar-35.fits.fz")] 

356 self.assertEqual(loc.getStorage().root, expectedRoot) 

357 self.assertEqual(loc.getLocations(), expectedLocations) 

358 

359 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

360 image = butler.get("someFz", ccd=35) 

361 self.assertEqual(image.getFilter().getName(), "r") 

362 self.assertEqual(image.getFilterLabel().bandLabel, "r") 

363 

364 bbox = geom.BoxI(geom.Point2I(200, 100), 

365 geom.Extent2I(300, 400)) 

366 image = butler.get("someFz_sub", ccd=35, bbox=bbox, imageOrigin="LOCAL", immediate=True) 

367 self.assertEqual(image.getHeight(), 400) 

368 self.assertEqual(image.getWidth(), 300) 

369 

370 def testButlerQueryMetadata(self): 

371 mapper = MinMapper2(root=ROOT) 

372 butler = dafPersist.ButlerFactory(mapper=mapper).create() 

373 kwargs = {"ccd": 35, "filter": "r", "visit": 787731, 

374 "taiObs": "2005-04-02T09:24:49.933440000"} 

375 self.assertEqual(butler.queryMetadata("other", "visit", **kwargs), [787731]) 

376 self.assertEqual(butler.queryMetadata("other", "visit", 

377 visit=kwargs["visit"], ccd=kwargs["ccd"], 

378 taiObs=kwargs["taiObs"], filter=kwargs["filter"]), 

379 [787731]) 

380 # now test we get no matches if ccd is out of range 

381 self.assertEqual(butler.queryMetadata("raw", "ccd", ccd=36, filter="r", visit=787731), []) 

382 

383 def testQueryMetadata(self): 

384 mapper = MinMapper2(root=ROOT) 

385 self.assertEqual(mapper.queryMetadata("raw", ["ccd"], None), 

386 [(x,) for x in range(36) if x != 3]) 

387 

388 def testStandardize(self): 

389 mapper = MinMapper2(root=ROOT) 

390 self.assertEqual(mapper.canStandardize("raw"), True) 

391 self.assertEqual(mapper.canStandardize("notPresent"), False) 

392 

393 def testStandardizeFiltersFilterDefs(self): 

394 testLabels = [ 

395 (None, None), 

396 (afwImage.FilterLabel(band="r", physical="r.MP9601"), 

397 afwImage.FilterLabel(band="r", physical="r.MP9601")), 

398 (afwImage.FilterLabel(band="r"), afwImage.FilterLabel(band="r", physical="r.MP9601")), 

399 (afwImage.FilterLabel(physical="r.MP9601"), 

400 afwImage.FilterLabel(band="r", physical="r.MP9601")), 

401 (afwImage.FilterLabel(band="r", physical="old-r"), 

402 afwImage.FilterLabel(band="r", physical="r.MP9601")), 

403 (afwImage.FilterLabel(physical="old-r"), 

404 afwImage.FilterLabel(band="r", physical="r.MP9601")), 

405 (afwImage.FilterLabel(physical="r2"), afwImage.FilterLabel(band="r", physical="HSC-R2")), 

406 ] 

407 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

408 "r", "r.MP9601", "old-r", "r2", 

409 }] 

410 testData = [] 

411 # Resolve special combinations where the expected output is different 

412 for input, corrected in testLabels: 

413 for dataId in testIds: 

414 if input is None: 

415 if dataId["filter"] == "r": 

416 data = (input, dataId, afwImage.FilterLabel(band="r")) 

417 elif dataId["filter"] == "r2": 

418 data = (input, dataId, afwImage.FilterLabel(band="r", physical="HSC-R2")) 

419 else: 

420 data = (input, dataId, afwImage.FilterLabel(band="r", physical="r.MP9601")) 

421 elif input == afwImage.FilterLabel(band="r"): 

422 if dataId["filter"] == "r": 

423 # There are two "r" filters, can't tell which 

424 data = (input, dataId, input) 

425 elif dataId["filter"] == "r2": 

426 data = (input, dataId, afwImage.FilterLabel(band="r", physical="HSC-R2")) 

427 elif corrected.physicalLabel == "HSC-R2" and dataId["filter"] in ("r.MP9601", "old-r"): 

428 # Contradictory inputs, leave as-is 

429 data = (input, dataId, input) 

430 elif corrected.physicalLabel == "r.MP9601" and dataId["filter"] == "r2": 

431 # Contradictory inputs, leave as-is 

432 data = (input, dataId, input) 

433 else: 

434 data = (input, dataId, corrected) 

435 testData.append(data) 

436 

437 mapper = MinMapper2(root=ROOT) 

438 for label, dataId, corrected in testData: 

439 exposure = afwImage.ExposureF() 

440 exposure.setFilterLabel(label) 

441 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

442 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

443 

444 def testStandardizeFiltersFilterNoDefs(self): 

445 testLabels = [ 

446 None, 

447 afwImage.FilterLabel(band="r", physical="r.MP9601"), 

448 afwImage.FilterLabel(band="r"), 

449 afwImage.FilterLabel(physical="r.MP9601"), 

450 afwImage.FilterLabel(band="r", physical="old-r"), 

451 afwImage.FilterLabel(physical="old-r"), 

452 afwImage.FilterLabel(physical="r2"), 

453 ] 

454 testIds = [{"visit": 12345, "ccd": 42, "filter": f} for f in { 

455 "r", "r.MP9601", "old-r", "r2", 

456 }] 

457 testData = [] 

458 # Resolve special combinations where the expected output is different 

459 for input in testLabels: 

460 for dataId in testIds: 

461 if input is None: 

462 # Can still get some filter info out of the Filter registry 

463 if dataId["filter"] == "r2": 

464 data = (input, dataId, 

465 afwImage.FilterLabel(band="r", physical="HSC-R2")) 

466 else: 

467 # Data ID maps to filter(s) with aliases; can't 

468 # unambiguously determine physical filter. 

469 data = (input, dataId, afwImage.FilterLabel(band="r")) 

470 else: 

471 data = (input, dataId, input) 

472 testData.append(data) 

473 

474 mapper = MinMapper1(root=ROOT) 

475 for label, dataId, corrected in testData: 

476 exposure = afwImage.ExposureF() 

477 exposure.setFilterLabel(label) 

478 mapper._setFilter(mapper.exposures['raw'], exposure, dataId) 

479 self.assertEqual(exposure.getFilterLabel(), corrected, msg=f"Started from {label} and {dataId}") 

480 

481 def testCalib(self): 

482 mapper = MinMapper2(root=ROOT) 

483 loc = mapper.map("flat", {"visit": 787650, "ccd": 13}, write=True) 

484 self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureF") 

485 self.assertEqual(loc.getCppType(), "ExposureF") 

486 self.assertEqual(loc.getStorageName(), "FitsStorage") 

487 expectedRoot = ROOT 

488 expectedLocations = ["flat-05Am03-fi.fits"] 

489 self.assertEqual(loc.getStorage().root, expectedRoot) 

490 self.assertEqual(loc.getLocations(), expectedLocations) 

491 self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13) 

492 self.assertEqual(loc.getAdditionalData().getScalar("visit"), 787650) 

493 self.assertEqual(loc.getAdditionalData().getScalar("derivedRunId"), "05Am03") 

494 self.assertEqual(loc.getAdditionalData().getScalar("filter"), "i") 

495 checkCompression(self, loc.getAdditionalData()) 

496 

497 def testNames(self): 

498 self.assertEqual(MinMapper2.getCameraName(), "min") 

499 self.assertEqual(MinMapper2.getPackageName(), "moe") 

500 

501 @unittest.expectedFailure 

502 def testParentSearch(self): 

503 mapper = MinMapper2(root=ROOT) 

504 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

505 os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))) 

506 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits'))]) 

507 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

508 os.path.join(ROOT, 

509 os.path.join('testParentSearch', 'bar.fits[1]'))) 

510 self.assertEqual(paths, [os.path.join(ROOT, os.path.join('testParentSearch', 'bar.fits[1]'))]) 

511 

512 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

513 os.path.join(ROOT, os.path.join('testParentSearch', 'baz.fits'))) 

514 self.assertEqual(paths, [os.path.join(ROOT, 

515 os.path.join('testParentSearch', '_parent', 'baz.fits'))]) 

516 paths = mapper.parentSearch(os.path.join(ROOT, 'testParentSearch'), 

517 os.path.join(ROOT, 

518 os.path.join('testParentSearch', 'baz.fits[1]'))) 

519 self.assertEqual(paths, [os.path.join(ROOT, 

520 os.path.join('testParentSearch', '_parent', 'baz.fits[1]'))]) 

521 

522 def testSkymapLookups(self): 

523 """Test that metadata lookups don't try to get skymap data ID values 

524 from the registry. 

525 """ 

526 mapper = MinMapper2(root=ROOT) 

527 butler = dafPersist.Butler(mapper=mapper) 

528 with self.assertRaises(RuntimeError) as manager: 

529 butler.dataRef("forced_src", visit=787650, ccd=13) 

530 self.assertIn("Cannot lookup skymap key 'tract'", str(manager.exception)) 

531 # We're mostly concerned that the statements below will raise an 

532 # exception; if they don't, it's not likely the following tests will 

533 # fail. 

534 subset = butler.subset("forced_src", visit=787650, ccd=13, tract=0) 

535 self.assertEqual(len(subset), 1) 

536 dataRef = butler.dataRef("forced_src", visit=787650, ccd=13, tract=0) 

537 self.assertFalse(dataRef.datasetExists("forced_src")) 

538 

539 

540class Mapper3TestCase(unittest.TestCase): 

541 """A test case for a mapper subclass which does not assign packageName.""" 

542 

543 def testPackageName(self): 

544 with self.assertRaises(ValueError): 

545 MinMapper3() 

546 with self.assertRaises(ValueError): 

547 MinMapper3.getPackageName() 

548 

549 

550class ParentRegistryTestCase(unittest.TestCase): 

551 

552 @staticmethod 

553 def _createRegistry(path): 

554 cmd = """CREATE TABLE x( 

555 id INT, 

556 visit INT, 

557 filter TEXT, 

558 snap INT, 

559 raft TEXT, 

560 sensor TEXT, 

561 channel TEXT, 

562 taiObs TEXT, 

563 expTime REAL 

564 ); 

565 """ 

566 conn = sqlite3.connect(path) 

567 conn.cursor().execute(cmd) 

568 conn.commit() 

569 conn.close() 

570 

571 def setUp(self): 

572 self.ROOT = tempfile.mkdtemp(dir=ROOT, prefix="ParentRegistryTestCase-") 

573 self.repoARoot = os.path.join(self.ROOT, 'a') 

574 args = dafPersist.RepositoryArgs(root=self.repoARoot, mapper=MinMapper1) 

575 butler = dafPersist.Butler(outputs=args) 

576 self._createRegistry(os.path.join(self.repoARoot, 'registry.sqlite3')) 

577 del butler 

578 

579 def tearDown(self): 

580 # the butler sql registry closes its database connection in __del__. 

581 # To trigger __del__ we explicitly collect the garbage here. If we 

582 # find having or closing the open database connection is a problem in 

583 # production code, we may need to add api to butler to explicity 

584 # release database connections (and maybe other things like in-memory 

585 # cached objects). 

586 gc.collect() 

587 if os.path.exists(self.ROOT): 

588 shutil.rmtree(self.ROOT) 

589 

590 def test(self): 

591 """Verify that when the child repo does not have a registry it is 

592 assigned the registry from the parent. 

593 """ 

594 repoBRoot = os.path.join(self.ROOT, 'b') 

595 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

596 # This way of getting the registry from the mapping is obviously going 

597 # way into private members and the python lambda implementation code. 

598 # It is very brittle and should not be duplicated in user code 

599 # or any location that is not trivial to fix along with changes to the 

600 # CameraMapper or Mapping. 

601 registryA = butler._repos.inputs()[0].repo._mapper.registry 

602 registryB = butler._repos.outputs()[0].repo._mapper.registry 

603 self.assertEqual(id(registryA), id(registryB)) 

604 

605 self._createRegistry(os.path.join(repoBRoot, 'registry.sqlite3')) 

606 butler = dafPersist.Butler(inputs=self.repoARoot, outputs=repoBRoot) 

607 # see above; don't copy this way of getting the registry. 

608 registryA = butler._repos.inputs()[0].repo._mapper.registry 

609 registryB = butler._repos.outputs()[0].repo._mapper.registry 

610 self.assertNotEqual(id(registryA), id(registryB)) 

611 

612 

613class MissingPolicyKeyTestCase(unittest.TestCase): 

614 

615 def testGetRaises(self): 

616 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

617 # MinMapper1 does not specify a template for the raw dataset type so 

618 # trying to use it for get should raise 

619 with self.assertRaises(RuntimeError) as contextManager: 

620 butler.get('raw') 

621 # This test demonstrates and verifies that simple use of the incomplete 

622 # dataset type returns a helpful (I hope) error message. 

623 self.assertEqual( 

624 str(contextManager.exception), 

625 'Template is not defined for the raw dataset type, ' 

626 'it must be set before it can be used.') 

627 with self.assertRaises(RuntimeError) as contextManager: 

628 butler.queryMetadata('raw', 'unused', {}) 

629 

630 def testQueryMetadataRaises(self): 

631 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

632 # MinMapper1 does not specify a template for the raw dataset type so 

633 # trying to use it for queryMetadata should raise 

634 with self.assertRaises(RuntimeError) as contextManager: 

635 butler.queryMetadata('raw', 'unused', {}) 

636 # This test demonstrates and verifies that simple use of the incomplete 

637 # dataset type returns a helpful (I hope) error message. 

638 self.assertEqual( 

639 str(contextManager.exception), 

640 'Template is not defined for the raw dataset type, ' 

641 'it must be set before it can be used.') 

642 

643 def testFilenameRaises(self): 

644 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

645 # MinMapper1 does not specify a template for the raw dataset type so 

646 # trying to use it for <datasetType>_filename should raise 

647 with self.assertRaises(RuntimeError) as contextManager: 

648 butler.get('raw_filename') 

649 # This test demonstrates and verifies that simple use of the 

650 # incomplete dataset type returns a helpful (I hope) error message. 

651 self.assertEqual( 

652 str(contextManager.exception), 

653 'Template is not defined for the raw dataset type, ' 

654 'it must be set before it can be used.') 

655 

656 def testWcsRaises(self): 

657 butler = dafPersist.Butler(inputs={'root': ROOT, 'mapper': MinMapper1}) 

658 # MinMapper1 does not specify a template for the raw dataset type so 

659 # trying to use it for <datasetType>_wcs should raise 

660 with self.assertRaises(RuntimeError) as contextManager: 

661 butler.get('raw_wcs') 

662 # This test demonstrates and verifies that simple use of the 

663 # incomplete dataset type returns a helpful (I hope) error message. 

664 self.assertEqual( 

665 str(contextManager.exception), 

666 'Template is not defined for the raw dataset type, ' 

667 'it must be set before it can be used.') 

668 

669 def testConflictRaises(self): 

670 policy = dafPersist.Policy(os.path.join(ROOT, "ConflictMapper.yaml")) 

671 with self.assertRaisesRegex( 

672 ValueError, 

673 r"Duplicate mapping policy for dataset type packages"): 

674 mapper = lsst.obs.base.CameraMapper(policy=policy, repositoryDir=ROOT, root=ROOT) # noqa F841 

675 

676 

677class MemoryTester(lsst.utils.tests.MemoryTestCase): 

678 pass 

679 

680 

681if __name__ == '__main__': 681 ↛ 682line 681 didn't jump to line 682, because the condition on line 681 was never true

682 lsst.utils.tests.init() 

683 unittest.main()