Coverage for tests/test_repository.py: 16%

572 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-03-20 00:41 -0700

1# -*- coding: UTF-8 -*- 

2 

3# 

4# LSST Data Management System 

5# Copyright 2016 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24 

25import os 

26import astropy.io.fits 

27import shutil 

28import sqlite3 

29import unittest 

30import tempfile 

31 

32import lsst.daf.persistence as dp 

33# can't use name TestObject, becuase it messes Pytest up. Alias it to tstObj 

34from lsst.daf.persistence.test import TestObject as tstObj 

35from lsst.daf.persistence.test import MapperForTestWriting 

36import lsst.utils.tests 

37 

38# Define the root of the tests relative to this file 

39ROOT = os.path.abspath(os.path.dirname(__file__)) 

40 

41 

42def setup_module(module): 

43 lsst.utils.tests.init() 

44 

45 

46class ParentMapper(dp.Mapper): 

47 

48 def __init__(self, root, **kwargs): 

49 self.root = root 

50 self.storage = dp.Storage.makeFromURI(self.root) 

51 

52 def __repr__(self): 

53 return 'ParentMapper(root=%s)' % self.root 

54 

55 def map_raw(self, dataId, write): 

56 python = 'astropy.io.fits.HDUList' 

57 persistable = None 

58 storage = 'PickleStorage' 

59 path = os.path.join(self.root, 'raw') 

60 path = os.path.join(path, 'raw_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz') 

61 if os.path.exists(path): 

62 return dp.ButlerLocation(python, persistable, storage, path, 

63 dataId, self, self.storage) 

64 return None 

65 

66 def bypass_raw(self, datasetType, pythonType, location, dataId): 

67 return astropy.io.fits.open(location.getLocations()[0]) 

68 

69 def query_raw(self, format, dataId): 

70 values = [{'visit': 1, 'filter': 'g'}, {'visit': 2, 'filter': 'g'}, {'visit': 3, 'filter': 'r'}] 

71 matches = [] 

72 for value in values: 

73 match = True 

74 for item in dataId: 

75 if value[item] != dataId[item]: 

76 match = False 

77 break 

78 if match: 

79 matches.append(value) 

80 results = set() 

81 for match in matches: 

82 tempTup = [] 

83 for word in format: 

84 tempTup.append(match[word]) 

85 results.add(tuple(tempTup)) 

86 return results 

87 

88 def getDefaultLevel(self): 

89 return 'visit' 

90 

91 def getKeys(self, datasetType, level): 

92 return {'filter': str, 'visit': int} 

93 

94 def map_str(self, dataId, write): 

95 path = os.path.join(self.root, 'raw') 

96 path = os.path.join(path, 'raw_v' + str(dataId['str']) + '_f' + dataId['filter'] + '.fits.gz') 

97 if os.path.exists(path): 

98 return dp.ButlerLocation(str, None, 'PickleStorage', path, dataId, 

99 self, self.storage) 

100 return None 

101 

102 

103class ChildrenMapper(dp.Mapper): 

104 

105 def __init__(self, root, **kwargs): 

106 self.root = root 

107 self.storage = dp.Storage.makeFromURI(self.root) 

108 

109 def map_raw(self, dataId, write): 

110 python = 'astropy.io.fits.HDUList' 

111 persistable = None 

112 storage = 'FitsStorage' 

113 path = os.path.join(self.root, 'raw') 

114 path = os.path.join(path, 'raw_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz') 

115 if write or os.path.exists(path): 

116 return dp.ButlerLocation(python, persistable, storage, path, 

117 dataId, self, self.storage) 

118 return None 

119 

120 def map_pickled(self, dataId, write): 

121 python = 'dict' 

122 persistable = None 

123 storage = 'PickleStorage' 

124 path = os.path.join(self.root, 'raw') 

125 path = os.path.join(path, 'pickled_v' + str(dataId['visit']) + '_f' + dataId['filter'] + '.fits.gz') 

126 if write or os.path.exists(path): 

127 return dp.ButlerLocation(python, persistable, storage, path, 

128 dataId, self, self.storage) 

129 

130 def bypass_raw(self, datasetType, pythonType, location, dataId): 

131 return astropy.io.fits.open(location.getLocations()[0]) 

132 

133 def query_raw(self, format, dataId): 

134 return None 

135 # results = set() 

136 # return results 

137 

138 def getDefaultLevel(self): 

139 return 'visit' 

140 

141 def getKeys(self, datasetType, level): 

142 return {'filter': str, 'visit': int} 

143 

144 

145class TestBasics(unittest.TestCase): 

146 """Test case for basic functions of the repository classes.""" 

147 

148 def setUp(self): 

149 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestBasics-") 

150 inputRepoArgs = dp.RepositoryArgs(root=os.path.join(ROOT, 'butlerAlias', 'data', 'input'), 

151 mapper=ParentMapper, 

152 tags='baArgs') 

153 # mode of output repos is write-only by default 

154 outputRepoArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'), 

155 mapper=ChildrenMapper, 

156 mode='rw') 

157 self.butler = dp.Butler(inputs=inputRepoArgs, outputs=outputRepoArgs) 

158 

159 def tearDown(self): 

160 if os.path.exists(self.testDir): 

161 shutil.rmtree(self.testDir) 

162 if os.path.exists(os.path.join(ROOT, 'butlerAlias/repositoryCfg.yaml')): 

163 os.remove(os.path.join(ROOT, 'butlerAlias/repositoryCfg.yaml')) 

164 if hasattr(self, "butler"): 

165 del self.butler 

166 

167 def testGet(self): 

168 raw_image = self.butler.get('raw', {'visit': '2', 'filter': 'g'}) 

169 # in this case the width is known to be 1026: 

170 self.assertEqual(raw_image[1].header["NAXIS1"], 1026) 

171 

172 def testGetUri(self): 

173 raw_uri = self.butler.getUri('raw', {'visit': '2', 'filter': 'g'}) 

174 self.assertEqual(raw_uri, 

175 os.path.join(ROOT, 'butlerAlias', 'data', 'input', 'raw', 'raw_v2_fg.fits.gz')) 

176 self.assertEqual(os.path.isfile(raw_uri), True) 

177 

178 def testGetUriWrite(self): 

179 raw_uri = self.butler.getUri('raw', {'visit': '9', 'filter': 'g'}, write=True) 

180 self.assertEqual(raw_uri, 

181 os.path.join(self.testDir, 'repoA', 'raw', 'raw_v9_fg.fits.gz')) 

182 self.assertEqual(os.path.isfile(raw_uri), False) 

183 

184 def testSubset(self): 

185 subset = self.butler.subset('raw') 

186 self.assertEqual(len(subset), 3) 

187 

188 def testGetKeys(self): 

189 keys = self.butler.getKeys('raw') 

190 self.assertIn('filter', keys) 

191 self.assertIn('visit', keys) 

192 self.assertEqual(keys['filter'], str) 

193 self.assertEqual(keys['visit'], int) 

194 

195 keys = self.butler.getKeys('raw', tag='baArgs') 

196 self.assertIn('filter', keys) 

197 self.assertIn('visit', keys) 

198 self.assertEqual(keys['filter'], str) 

199 self.assertEqual(keys['visit'], int) 

200 

201 keys = self.butler.getKeys('raw', tag=('baArgs', 'foo')) 

202 self.assertIn('filter', keys) 

203 self.assertIn('visit', keys) 

204 self.assertEqual(keys['filter'], str) 

205 self.assertEqual(keys['visit'], int) 

206 

207 keys = self.butler.getKeys('raw', tag='foo') 

208 self.assertIsNone(keys) 

209 

210 keys = self.butler.getKeys('raw', tag=set(['baArgs', 'foo'])) 

211 self.assertIn('filter', keys) 

212 self.assertIn('visit', keys) 

213 self.assertEqual(keys['filter'], str) 

214 self.assertEqual(keys['visit'], int) 

215 

216 def testGetDatasetTypes(self): 

217 self.assertEqual(self.butler.getDatasetTypes(), {"raw", "str", "pickled"}) 

218 

219 def testQueryMetadata(self): 

220 val = self.butler.queryMetadata('raw', ('filter',)) 

221 val.sort() 

222 self.assertEqual(val, ['g', 'r']) 

223 

224 val = self.butler.queryMetadata('raw', ('visit',)) 

225 val.sort() 

226 self.assertEqual(val, [1, 2, 3]) 

227 

228 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'g'}) 

229 val.sort() 

230 self.assertEqual(val, [1, 2]) 

231 

232 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'r'}) 

233 self.assertEqual(val, [3]) 

234 

235 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 1}) 

236 self.assertEqual(val, ['g']) 

237 

238 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 2}) 

239 self.assertEqual(val, ['g']) 

240 

241 val = self.butler.queryMetadata('raw', ('filter',), dataId={'visit': 3}) 

242 self.assertEqual(val, ['r']) 

243 

244 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'h'}) 

245 self.assertEqual(val, []) 

246 

247 dataId = dp.DataId({'filter': 'g'}, tag='baArgs') 

248 val = self.butler.queryMetadata('raw', ('visit',), dataId={'filter': 'g'}) 

249 val.sort() 

250 self.assertEqual(val, [1, 2]) 

251 

252 dataId = dp.DataId({'filter': 'g'}, tag='foo') 

253 val = self.butler.queryMetadata('raw', ('visit',), dataId=dataId) 

254 self.assertEqual(val, []) 

255 

256 def testDatasetExists(self): 

257 # test the values that are expected to be true: 

258 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 1}), True) 

259 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 2}), True) 

260 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 3}), True) 

261 

262 # test a few values that are expected to be false: 

263 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'f', 'visit': 1}), False) 

264 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 1}), False) 

265 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 3}), False) 

266 

267 # test that we don't see datasets in the input repo as being in the output repo 

268 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 1}, write=True), False) 

269 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'g', 'visit': 2}, write=True), False) 

270 self.assertEqual(self.butler.datasetExists('raw', {'filter': 'r', 'visit': 3}, write=True), False) 

271 

272 # write a dataset to the output repo and check that we see it with both write=True and write=False 

273 dataId = {'visit': '2', 'filter': 'g'} 

274 self.butler.put({"number": 3}, 'pickled', dataId) 

275 self.assertEqual(self.butler.datasetExists('pickled', dataId, write=True), True) 

276 self.assertEqual(self.butler.datasetExists('pickled', dataId, write=False), True) 

277 

278 

279############################################################################################################## 

280############################################################################################################## 

281############################################################################################################## 

282 

283 

284class NoResultsMapper(dp.Mapper): 

285 

286 def __init__(self, root, **kwargs): 

287 self.root = root 

288 self.storage = dp.Storage.makeFromURI(self.root) 

289 

290 def __repr__(self): 

291 return 'ParentMapper(root=%s)' % self.root 

292 

293 def map_list(self, dataId, write): 

294 return [] 

295 

296 def map_None(self, dataId, write): 

297 return None 

298 

299 

300class TestWriting(unittest.TestCase): 

301 """A test case for the repository classes. 

302 

303 A test that 

304 1. creates repo with a peer repo, writes to those repos. 

305 2. reloads those output repos as a parents of new repos 

306 * does a read from from the repo (verifies parent search) 

307 3. writes to the new output repo and reloads it as a parent of a new repo 

308 * verifies masking 

309 4. reloads the repo from its persisted cfg 

310 * verifies reload from cfg 

311 """ 

312 def setUp(self): 

313 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestWriting-") 

314 

315 def tearDown(self): 

316 if os.path.exists(self.testDir): 

317 shutil.rmtree(self.testDir) 

318 # del self.butler 

319 

320 def testCreateAggregateAndLoadingAChild(self): 

321 """Tests putting a very basic pickled object in a variety of Repository configuration settings 

322 :return: 

323 """ 

324 

325 repoAArgs = dp.RepositoryArgs(mode='w', 

326 root=os.path.join(self.testDir, 'repoA'), 

327 mapper=MapperForTestWriting) 

328 repoBArgs = dp.RepositoryArgs(mode='w', 

329 root=os.path.join(self.testDir, 'repoB'), 

330 mapper=MapperForTestWriting) 

331 butlerAB = dp.Butler(outputs=[repoAArgs, repoBArgs]) 

332 

333 objA = tstObj('abc') 

334 butlerAB.put(objA, 'foo', {'val': 1}) 

335 objB = tstObj('def') 

336 butlerAB.put(objB, 'foo', {'val': 2}) 

337 

338 # create butlers where the output repos are now input repos 

339 butlerC = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'))) 

340 butlerD = dp.Butler(inputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB'))) 

341 

342 # # verify the objects exist by getting them 

343 self.assertEqual(objA, butlerC.get('foo', {'val': 1})) 

344 self.assertEqual(objA, butlerC.get('foo', {'val': 1})) 

345 self.assertEqual(objB, butlerD.get('foo', {'val': 2})) 

346 self.assertEqual(objB, butlerD.get('foo', {'val': 2})) 

347 

348 def testPutWithIncompleteDataId(self): 

349 repoAArgs = dp.RepositoryArgs(mode='rw', 

350 root=os.path.join(self.testDir, 'repoA'), 

351 mapper=NoResultsMapper) 

352 butler = dp.Butler(outputs=repoAArgs) 

353 obj = tstObj('abc') 

354 with self.assertRaises(RuntimeError): 

355 butler.put(obj, 'list', {'visit': '2'}) 

356 with self.assertRaises(RuntimeError): 

357 butler.put(obj, 'None', {'visit': '2'}) 

358 

359 

360class TestDiamondPattern(unittest.TestCase): 

361 """A test case for when a butler is created with an output repository and two input repositories that have 

362 a common parent; verifies that the parent's registry is loaded as the parentRegistry of the output 

363 repository. 

364 """ 

365 @staticmethod 

366 def makeRegistry(location, name): 

367 conn = sqlite3.connect(location) 

368 conn.execute("CREATE TABLE {name} (real)".format(name=name)) 

369 conn.commit() 

370 conn.close() 

371 

372 def setUp(self): 

373 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestDiamondPattern-") 

374 

375 def tearDown(self): 

376 if os.path.exists(self.testDir): 

377 shutil.rmtree(self.testDir) 

378 

379 def test(self): 

380 repoARoot = os.path.join(self.testDir, 'repoA') 

381 butler = dp.Butler(outputs={'root': repoARoot, 'mapper': ParentRepoTestMapper}) 

382 self.makeRegistry(os.path.join(repoARoot, 'registry.sqlite3'), 'repoA') 

383 del butler 

384 

385 repoBRoot = os.path.join(self.testDir, 'repoB') 

386 butlerB = dp.Butler(inputs=repoARoot, outputs=repoBRoot) 

387 del butlerB 

388 

389 repoCRoot = os.path.join(self.testDir, 'repoC') 

390 butlerC = dp.Butler(inputs=repoARoot, outputs=repoCRoot) 

391 del butlerC 

392 

393 repoDRoot = os.path.join(self.testDir, 'repoD') 

394 butlerD = dp.Butler(inputs=(repoBRoot, repoCRoot), outputs=repoDRoot) 

395 

396 self.assertEqual(1, len(butlerD._repos.outputs())) 

397 self.assertEqual(os.path.dirname(butlerD._repos.outputs()[0].parentRegistry.root), repoARoot) 

398 

399 

400class TestMasking(unittest.TestCase): 

401 """A test case for the repository classes. 

402 

403 A test that 

404 1. creates a repo, does a put 

405 2. creates a new butler, uses that repo as an input and creates a new read-write output repo 

406 3. gets from the input repo, modifies the dataset, and puts into the output repo 

407 4. does a get and verifies that the changed dataset is returned. 

408 """ 

409 def setUp(self): 

410 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMasking-") 

411 

412 def tearDown(self): 

413 if os.path.exists(self.testDir): 

414 shutil.rmtree(self.testDir) 

415 

416 def test(self): 

417 repoAArgs = dp.RepositoryArgs(mode='w', 

418 root=os.path.join(self.testDir, 'repoA'), 

419 mapper=MapperForTestWriting) 

420 butler = dp.Butler(outputs=repoAArgs) 

421 obj0 = tstObj('abc') 

422 butler.put(obj0, 'foo', {'bar': 1}) 

423 del butler 

424 del repoAArgs 

425 

426 repoBArgs = dp.RepositoryArgs(mode='rw', 

427 root=os.path.join(self.testDir, 'repoB'), 

428 mapper=MapperForTestWriting) 

429 butler = dp.Butler(inputs=os.path.join(self.testDir, 'repoA'), outputs=repoBArgs) 

430 obj1 = butler.get('foo', {'bar': 1}) 

431 self.assertEqual(obj0, obj1) 

432 obj1.data = "def" 

433 butler.put(obj1, 'foo', {'bar': 1}) 

434 obj2 = butler.get('foo', {'bar': 1}) 

435 self.assertEqual(obj1, obj2) 

436 

437 

438class TestMultipleOutputsPut(unittest.TestCase): 

439 """A test case for the repository classes. 

440 

441 A test that 

442 1. creates 3 peer repositories and readers for them 

443 2. does a single put 

444 3. verifies that all repos received the put 

445 """ 

446 def setUp(self): 

447 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMultipleOutputsPut-") 

448 

449 def tearDown(self): 

450 if os.path.exists(self.testDir): 

451 shutil.rmtree(self.testDir) 

452 

453 def test(self): 

454 repoAArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoA'), 

455 mapper=MapperForTestWriting) 

456 repoBArgs = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repoB'), 

457 mapper=MapperForTestWriting) 

458 

459 butler = dp.Butler(outputs=(repoAArgs, repoBArgs)) 

460 obj0 = tstObj('abc') 

461 butler.put(obj0, 'foo', {'bar': 1}) 

462 

463 for root in (os.path.join(self.testDir, 'repoA'), 

464 os.path.join(self.testDir, 'repoB')): 

465 butler = dp.Butler(inputs=root) 

466 self.assertEqual(butler.get('foo', {'bar': 1}), obj0) 

467 

468 

469class TestMultipleInputs(unittest.TestCase): 

470 """A test case for the repository classes. 

471 

472 A test that 

473 - create output 2 repos, write same & different data to them & close them 

474 - create a new butler with those repos as inputs 

475 - read data that was written to both repos: 

476 - verify data that was written to only one of each repo 

477 - verify dissimilar datasets with same id that were written to the repos 

478 

479 """ 

480 def setUp(self): 

481 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMapperInference-") 

482 

483 def tearDown(self): 

484 if os.path.exists(self.testDir): 

485 shutil.rmtree(self.testDir) 

486 

487 def test(self): 

488 objAbc = tstObj('abc') 

489 objDef = tstObj('def') 

490 objGhi = tstObj('ghi') 

491 objJkl = tstObj('jkl') 

492 

493 repoAArgs = dp.RepositoryArgs(mode='w', 

494 root=os.path.join(self.testDir, 'repoA'), 

495 mapper=MapperForTestWriting) 

496 butler = dp.Butler(outputs=repoAArgs) 

497 butler.put(objAbc, 'foo', {'bar': 1}) 

498 butler.put(objDef, 'foo', {'bar': 2}) 

499 

500 repoBArgs = dp.RepositoryArgs(mode='w', 

501 root=os.path.join(self.testDir, 'repoB'), 

502 mapper=MapperForTestWriting) 

503 butler = dp.Butler(outputs=repoBArgs) 

504 butler.put(objGhi, 'foo', {'bar': 1}) # note different object with overlapping dataId with repoA 

505 butler.put(objJkl, 'foo', {'bar': 3}) 

506 

507 # note repo order: A, B 

508 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'), 

509 os.path.join(self.testDir, 'repoB'))) 

510 self.assertEqual(butler.get('foo', {'bar': 1}), objAbc) 

511 self.assertEqual(butler.get('foo', {'bar': 2}), objDef) 

512 self.assertEqual(butler.get('foo', {'bar': 3}), objJkl) 

513 

514 # note reversed repo order: B, A 

515 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoB'), 

516 os.path.join(self.testDir, 'repoA'))) 

517 self.assertEqual(butler.get('foo', {'bar': 1}), objGhi) 

518 self.assertEqual(butler.get('foo', {'bar': 2}), objDef) 

519 self.assertEqual(butler.get('foo', {'bar': 3}), objJkl) 

520 

521 

522class TestTagging(unittest.TestCase): 

523 """A test case for the tagging of repository classes. 

524 """ 

525 

526 def setUp(self): 

527 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestTagging-") 

528 

529 def tearDown(self): 

530 if os.path.exists(self.testDir): 

531 shutil.rmtree(self.testDir) 

532 

533 def testOneLevelInputs(self): 

534 """ 

535 1. put an object with the same ID but slightly different value into 2 repositories. 

536 2. use those repositories as inputs to a butler, and tag them 

537 3. make sure that the correct object is gotten for each of 

538 a. one tag 

539 b. the other tag 

540 c. no tag 

541 4. repeat step 3 but reverse the order of input cfgs to a new butler. 

542 5. use the butler from step 4 and write an output. The inputs will get recorded as parents of the 

543 output repo. 

544 6. create a new butler with a new overlapping repo, and verify that objects can be gotten from the 

545 other's parent repos via tagging. 

546 """ 

547 objA = tstObj('a') 

548 objB = tstObj('b') 

549 

550 # put objA in repo1: 

551 repo1Args = dp.RepositoryArgs(mode='rw', 

552 root=os.path.join(self.testDir, 'repo1'), 

553 mapper=MapperForTestWriting) 

554 butler = dp.Butler(outputs=repo1Args) 

555 butler.put(objA, 'foo', {'bar': 1}) 

556 del butler 

557 

558 # put objB in repo2: 

559 repo2Args = dp.RepositoryArgs(mode='rw', 

560 root=os.path.join(self.testDir, 'repo2'), 

561 mapper=MapperForTestWriting) 

562 butler = dp.Butler(outputs=repo2Args) 

563 butler.put(objB, 'foo', {'bar': 1}) 

564 del butler 

565 del repo1Args 

566 del repo2Args 

567 

568 # make the objects inputs of repos 

569 # and verify the correct object can ge fetched using the tag and not using the tag 

570 

571 repo1Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo1'), tags='one') 

572 repo2Args = dp.RepositoryArgs(root=os.path.join(self.testDir, 'repo2'), tags='two') 

573 

574 butler = dp.Butler(inputs=(repo1Args, repo2Args)) 

575 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) 

576 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) 

577 self.assertEqual(butler.get('foo', {'bar': 1}), objA) 

578 

579 butler = dp.Butler(inputs=(repo2Args, repo1Args)) 

580 self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='one')), objA) 

581 self.assertEqual(butler.get('foo', dp.DataId(bar=1, tag='two')), objB) 

582 self.assertEqual(butler.get('foo', dp.DataId(bar=1)), objB) 

583 

584 # create butler with repo1 and repo2 as parents, and an output repo3. 

585 repo3Args = dp.RepositoryArgs(mode='rw', 

586 root=os.path.join(self.testDir, 'repo3'), 

587 mapper=MapperForTestWriting) 

588 butler = dp.Butler(inputs=(repo1Args, repo2Args), outputs=repo3Args) 

589 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) 

590 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) 

591 self.assertEqual(butler.get('foo', {'bar': 1}), objA) 

592 # add an object to the output repo. note since the output repo mode is 'rw' that object is gettable 

593 # and it has first priority in search order. Other repos should be searchable by tagging. 

594 objC = tstObj('c') 

595 butler.put(objC, 'foo', {'bar': 1}) 

596 self.assertEqual(butler.get('foo', {'bar': 1}), objC) 

597 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='one')), objA) 

598 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='two')), objB) 

599 del butler 

600 

601 repo3Cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'repo3')) 

602 self.assertEqual(repo3Cfg.parents, [os.path.join(self.testDir, 'repo1'), 

603 os.path.join(self.testDir, 'repo2')]) 

604 

605 # expand the structure to look like this: 

606 # ┌────────────────────────┐ ┌────────────────────────┐ 

607 # │repo1 │ │repo2 │ 

608 # │ tag:"one" │ │ tag:"two" │ 

609 # │ tstObj('a') │ │ tstObj('b') │ 

610 # │ at ('foo', {'bar:1'})│ │ at ('foo', {'bar:1'})│ 

611 # └───────────┬────────────┘ └───────────┬────────────┘ 

612 # └─────────────┬────────────┘ 

613 # ┌────────────┴───────────┐ ┌────────────────────────┐ 

614 # │repo4 │ │repo5 │ 

615 # │ tag:"four" │ │ tag:"five" │ 

616 # │ tstObj('d') │ │ tstObj('e') │ 

617 # │ at ('foo', {'bar:2'})│ │ at ('foo', {'bar:1'})│ 

618 # └───────────┬────────────┘ └───────────┬────────────┘ 

619 # └─────────────┬────────────┘ 

620 # ┌──┴───┐ 

621 # │butler│ 

622 # └──────┘ 

623 

624 repo4Args = dp.RepositoryArgs(mode='rw', 

625 root=os.path.join(self.testDir, 'repo4'), 

626 mapper=MapperForTestWriting) 

627 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repo1'), 

628 os.path.join(self.testDir, 'repo2')), outputs=repo4Args) 

629 objD = tstObj('d') 

630 butler.put(objD, 'foo', {'bar': 2}) 

631 del butler 

632 

633 repo5Cfg = dp.RepositoryArgs(mode='rw', 

634 root=os.path.join(self.testDir, 'repo5'), 

635 mapper=MapperForTestWriting) 

636 butler = dp.Butler(outputs=repo5Cfg) 

637 objE = tstObj('e') 

638 butler.put(objE, 'foo', {'bar': 1}) 

639 del butler 

640 

641 repo4Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo4'), tags='four') 

642 repo5Args = dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'repo5'), tags='five') 

643 butler = dp.Butler(inputs=(repo4Args, repo5Args)) 

644 self.assertEqual(butler.get('foo', {'bar': 1}), objA) 

645 self.assertEqual(butler.get('foo', {'bar': 2}), objD) 

646 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) 

647 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) 

648 del butler 

649 

650 butler = dp.Butler(inputs=(repo5Args, repo4Args)) 

651 self.assertEqual(butler.get('foo', {'bar': 1}), objE) 

652 self.assertEqual(butler.get('foo', {'bar': 2}), objD) 

653 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='four')), objA) 

654 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='five')), objE) 

655 del butler 

656 

657 

658class TestMapperInference(unittest.TestCase): 

659 """A test for inferring mapper in the cfg from parent cfgs""" 

660 

661 def setUp(self): 

662 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMapperInference-") 

663 

664 def tearDown(self): 

665 if os.path.exists(self.testDir): 

666 shutil.rmtree(self.testDir) 

667 

668 def testSingleParent(self): 

669 """ 

670 creates a repo that: 

671 a. does not have a mapper specified in the cfg 

672 b. has a parent that does have a mapper specified in the cfg 

673 verifies that the child repo inherits the parent's mapper via inference. 

674 """ 

675 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'), 

676 mapper=MapperForTestWriting, 

677 mapperArgs=None, 

678 parents=None, 

679 policy=None) 

680 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA')) 

681 

682 repoBArgs = dp.RepositoryArgs(mode='rw', 

683 root=os.path.join(self.testDir, 'repoB')) 

684 butler = dp.Butler(inputs=os.path.join(self.testDir, 'repoA'), outputs=repoBArgs) 

685 self.assertIsInstance(butler._repos.outputs()[0].repo._mapper, MapperForTestWriting) 

686 

687 def testMultipleParentsSameMapper(self): 

688 """ 

689 creates a repo that: 

690 a. does not have a mapper specified in the cfg 

691 b. has 2 parents that do have the same mapper specified in the cfg 

692 verifies that the child repo inherits the parent's mapper via inference. 

693 

694 """ 

695 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'), 

696 mapper=MapperForTestWriting, 

697 mapperArgs=None, 

698 parents=None, 

699 policy=None,) 

700 repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'), 

701 mapper=MapperForTestWriting, 

702 mapperArgs=None, 

703 parents=None, 

704 policy=None) 

705 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA')) 

706 dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB')) 

707 

708 repoCArgs = dp.RepositoryArgs(mode='w', 

709 root=os.path.join(self.testDir, 'repoC')) 

710 

711 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'repoA'), 

712 os.path.join(self.testDir, 'repoB')), 

713 outputs=repoCArgs) 

714 self.assertIsInstance(butler._repos.outputs()[0].repo._mapper, MapperForTestWriting) 

715 

716 def testMultipleParentsDifferentMappers(self): 

717 """ 

718 creates a repo that: 

719 a. does not have a mapper specified in the cfg 

720 b. has 2 parent repos that have different mappers 

721 verifies that the constructor raises a RuntimeError because the mapper can not be inferred. 

722 """ 

723 repoACfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoA'), 

724 mapper=MapperForTestWriting, 

725 mapperArgs=None, 

726 parents=None, 

727 policy=None) 

728 repoBCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'repoB'), 

729 mapper='lsst.daf.persistence.test.EmptyTestMapper', 

730 mapperArgs=None, 

731 parents=None, 

732 policy=None) 

733 dp.Storage.putRepositoryCfg(repoACfg, os.path.join(self.testDir, 'repoA')) 

734 dp.Storage.putRepositoryCfg(repoBCfg, os.path.join(self.testDir, 'repoB')) 

735 

736 repoCArgs = dp.RepositoryArgs(mode='w', 

737 root=os.path.join(self.testDir, 'repoC')) 

738 self.assertRaises(RuntimeError, 

739 dp.Butler, 

740 inputs=(os.path.join(self.testDir, 'repoA'), 

741 os.path.join(self.testDir, 'repoB')), 

742 outputs=repoCArgs) 

743 

744 

745class TestMovedRepositoryCfg(unittest.TestCase): 

746 """Test if a repository cfg is in-place (resides at root of the repository) and the cfg is moved, the 

747 deserialized cfg root should be the new location if the repository is moved. 

748 """ 

749 

750 def setUp(self): 

751 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestMovedRepositoryCfg-") 

752 

753 def tearDown(self): 

754 if os.path.exists(self.testDir): 

755 shutil.rmtree(self.testDir) 

756 

757 def test(self): 

758 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'), 

759 mapper=MapperForTestWriting)) 

760 del butler 

761 os.makedirs(os.path.join(self.testDir, 'b')) 

762 os.rename(os.path.join(self.testDir, 'a/repositoryCfg.yaml'), 

763 os.path.join(self.testDir, 'b/repositoryCfg.yaml')) 

764 butler = dp.Butler(inputs=os.path.join(self.testDir, 'b')) 

765 self.assertEqual(list(butler._repos.inputs())[0].cfg, 

766 dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), 

767 mapper=MapperForTestWriting, 

768 mapperArgs=None, 

769 parents=None, 

770 policy=None)) 

771 

772 

773class TestOutputAlreadyHasParent(unittest.TestCase): 

774 

775 def setUp(self): 

776 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOutputAlreadyHasParent-") 

777 

778 def tearDown(self): 

779 if os.path.exists(self.testDir): 

780 shutil.rmtree(self.testDir) 

781 

782 def test(self): 

783 # create a repo where repo 'a' is a parent of repo 'b' 

784 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'a'), 

785 mapper=MapperForTestWriting)) 

786 del butler 

787 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

788 outputs=os.path.join(self.testDir, 'b')) 

789 self.assertEqual(len(butler._repos.inputs()), 1) 

790 self.assertEqual(butler._repos.inputs()[0].cfg.root, 

791 os.path.join(self.testDir, 'a')) 

792 self.assertEqual(len(butler._repos.outputs()), 1) 

793 self.assertEqual(butler._repos.outputs()[0].cfg.root, 

794 os.path.join(self.testDir, 'b')) 

795 del butler 

796 

797 # load that repo a few times, include 'a' as an input. 

798 for i in range(4): 

799 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

800 outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'b'), 

801 mode='rw')) 

802 self.assertEqual(len(butler._repos.inputs()), 2) 

803 self.assertEqual(butler._repos.inputs()[0].cfg.root, 

804 os.path.join(self.testDir, 'b')) 

805 self.assertEqual(butler._repos.inputs()[1].cfg.root, 

806 os.path.join(self.testDir, 'a')) 

807 self.assertEqual(len(butler._repos.outputs()), 1) 

808 self.assertEqual(butler._repos.outputs()[0].cfg.root, 

809 os.path.join(self.testDir, 'b')) 

810 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b')) 

811 self.assertEqual(cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), 

812 mapper=MapperForTestWriting, 

813 mapperArgs=None, 

814 parents=[os.path.join(self.testDir, 'a')], 

815 policy=None)) 

816 

817 # load the repo a few times and don't explicitly list 'a' as an input 

818 for i in range(4): 

819 butler = dp.Butler(outputs=dp.RepositoryArgs(root=os.path.join(self.testDir, 'b'), 

820 mode='rw')) 

821 self.assertEqual(len(butler._repos.inputs()), 2) 

822 self.assertEqual(butler._repos.inputs()[0].cfg.root, 

823 os.path.join(self.testDir, 'b')) 

824 self.assertEqual(butler._repos.inputs()[1].cfg.root, 

825 os.path.join(self.testDir, 'a')) 

826 self.assertEqual(len(butler._repos.outputs()), 1) 

827 self.assertEqual(butler._repos.outputs()[0].cfg.root, 

828 os.path.join(self.testDir, 'b')) 

829 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b')) 

830 self.assertEqual(cfg, dp.RepositoryCfg(root=os.path.join(self.testDir, 'b'), 

831 mapper=MapperForTestWriting, 

832 mapperArgs=None, 

833 parents=[os.path.join(self.testDir, 'a')], 

834 policy=None)) 

835 

836 # load 'b' as 'write only' and don't list 'a' as an input. This should raise, because inputs must 

837 # match readable outputs parents. 

838 with self.assertRaises(RuntimeError): 

839 butler = dp.Butler(outputs=os.path.join(self.testDir, 'b')) 

840 

841 # load 'b' as 'write only' and explicitly list 'a' as an input. 

842 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

843 outputs=os.path.join(self.testDir, 'b')) 

844 self.assertEqual(len(butler._repos.inputs()), 1) 

845 self.assertEqual(len(butler._repos.outputs()), 1) 

846 self.assertEqual(butler._repos.inputs()[0].cfg.root, 

847 os.path.join(self.testDir, 'a')) 

848 self.assertEqual(butler._repos.outputs()[0].cfg.root, 

849 os.path.join(self.testDir, 'b')) 

850 cfg = dp.Storage().getRepositoryCfg(os.path.join(self.testDir, 'b')) 

851 

852 

853class ParentRepoTestMapper(dp.Mapper): 

854 def __init__(self, parentRegistry, repositoryCfg, **kwargs): 

855 self.parentRegistry = parentRegistry 

856 root = repositoryCfg.root 

857 if os.path.exists(os.path.join(root, 'registry.sqlite3')): 

858 self.registry = dp.Registry.create(os.path.join(root, 'registry.sqlite3')) 

859 else: 

860 self.registry = None 

861 

862 def getRegistry(self): 

863 return self.registry 

864 

865 

866class TestParentRepository(unittest.TestCase): 

867 """A test to verify that when a parent repository is used the correct one is found.""" 

868 

869 def setUp(self): 

870 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestParentRepository-") 

871 

872 def tearDown(self): 

873 if os.path.exists(self.testDir): 

874 shutil.rmtree(self.testDir, True) 

875 

876 def test(self): 

877 

878 """Tests 1. That an sqlite registry in a parent repo is used as the 

879 registry in a child repo that does not have its own sqlite registry. 

880 2. That when a repo has a parent and grandparent and only the 

881 grandparent has an sqlite registry that the grandparent's registry is 

882 used. 

883 3. That when the parent and grandparent both have sqlite registries 

884 that the parent's sqlite registry is used.""" 

885 # setup; create the parent repo 

886 def makeRegistry(location, name): 

887 conn = sqlite3.connect(location) 

888 conn.execute("CREATE TABLE {name} (real)".format(name=name)) 

889 conn.commit() 

890 conn.close() 

891 

892 repoADir = os.path.join(self.testDir, 'repoA') 

893 repoBDir = os.path.join(self.testDir, 'repoB') 

894 repoCDir = os.path.join(self.testDir, 'repoC') 

895 butler = dp.Butler(outputs={'root': repoADir, 'mapper': ParentRepoTestMapper}) 

896 del butler 

897 makeRegistry(os.path.join(repoADir, 'registry.sqlite3'), 'repoA') 

898 # test 1: 

899 butler = dp.Butler(inputs=repoADir, outputs=repoBDir) 

900 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry 

901 self.assertIsInstance(registry, dp.SqliteRegistry) 

902 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() 

903 self.assertEqual(tables, [('repoA', )]) 

904 del butler 

905 # test 2: 

906 butler = dp.Butler(inputs=repoBDir, outputs=repoCDir) 

907 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry 

908 self.assertIsInstance(registry, dp.SqliteRegistry) 

909 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() 

910 self.assertEqual(tables, [('repoA', )]) 

911 del butler 

912 # test 3: 

913 makeRegistry(os.path.join(repoBDir, 'registry.sqlite3'), 'repoB') 

914 butler = dp.Butler(inputs=repoBDir, outputs=repoCDir) 

915 registry = butler._repos.outputs()[0].repo._mapper.parentRegistry 

916 self.assertIsInstance(registry, dp.SqliteRegistry) 

917 tables = registry.conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() 

918 self.assertEqual(tables, [('repoB', )]) 

919 

920 

921class TestOldButlerParent(unittest.TestCase): 

922 """A test to verify that when a parent is an old butler repo that it still gets loaded correctly, 

923 including mapperArgs.""" 

924 

925 def setUp(self): 

926 """Create temp test dir and create an OldButler repo at 'repoA'""" 

927 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOldButlerParent-") 

928 self.repoADir = os.path.join(self.testDir, 'repoA') 

929 os.makedirs(self.repoADir) 

930 with open(os.path.join(self.repoADir, '_mapper'), 'w') as mapperFile: 

931 mapperFile.write('lsst.daf.persistence.test.EmptyTestMapper') 

932 

933 def tearDown(self): 

934 if os.path.exists(self.testDir): 

935 shutil.rmtree(self.testDir, True) 

936 

937 def testOldButlerCfgRecordedInOutputs(self): 

938 """Verify that when an Old Butler is used as an input with parameters such as mapperArgs, that those 

939 parameters get recalled correctly (in the RepositoryCfg nested in the parents list of the output repo) 

940 """ 

941 repoBDir = os.path.join(self.testDir, 'repoB') 

942 # create a 'repoB' with the Old Butler 'repoA' as a parent, specify mapperArgs for 'repoA', and 

943 # verify that the mapperArgs got passed to the repoA mapper. 

944 butler = dp.Butler(inputs={'root': self.repoADir, 

945 'mapperArgs': {'foo': 1}}, 

946 outputs={'root': repoBDir, 

947 'mapperArgs': {'foo': 1}, 

948 'mode': 'rw'}) 

949 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1}) 

950 # init a Butler again, with exactly the same configuration as before. 

951 butler = dp.Butler(inputs={'root': self.repoADir, 

952 'mapperArgs': {'foo': 1}}, 

953 outputs={'root': repoBDir, 

954 'mapperArgs': {'foo': 1}, 

955 'mode': 'rw'}) 

956 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1}) 

957 # try initializing the butler again with the 'repoB' output, but do not specify the mapperArgs in the 

958 # repoA args. this should raise a runtime error because the input speficied this way does not match 

959 # the cfg recorded in 'repoB'. 

960 with self.assertRaises(RuntimeError): 

961 butler = dp.Butler(inputs=self.repoADir, 

962 outputs=repoBDir) 

963 # initializing the butler, but only specifiying the root of 'repoB' (the rest of the cfg will be 

964 # loaded from its `repositoryCfg` file), but fully specify the input as required. 

965 butler = dp.Butler(inputs={'root': self.repoADir, 

966 'mapperArgs': {'foo': 1}}, 

967 outputs=repoBDir) 

968 self.assertEqual(butler._repos.inputs()[0].repo._mapper.kwargs, {'foo': 1}) 

969 # use 'repoB' as an input, and verify that 'repoA' is loaded, including the mapperArgs. 

970 butler = dp.Butler(inputs=repoBDir) 

971 self.assertEqual(butler._repos.inputs()[1].repo._mapper.kwargs, {'foo': 1}) 

972 

973 

974class TestOldButlerParentTagging(unittest.TestCase): 

975 """A test to verify that when a parent is an old butler repo that its tagging gets loaded correctly""" 

976 

977 def setUp(self): 

978 """Remove testDir from any previous runs and create an OldButler repo at repoA and an Old Butler repoB 

979 with a _parent symlink to repoA. 

980 """ 

981 self.testDir = tempfile.mkdtemp(dir=ROOT, prefix="TestOldButlerParentTagging-") 

982 self.repoADir = os.path.join(self.testDir, 'repoA') 

983 os.makedirs(self.repoADir) 

984 with open(os.path.join(self.repoADir, '_mapper'), 'w') as mapperFile: 

985 mapperFile.write('lsst.daf.persistence.test.MapperForTestWriting') 

986 self.repoBDir = os.path.join(self.testDir, 'repoB') 

987 os.makedirs(self.repoBDir) 

988 os.symlink(self.repoADir, os.path.join(self.repoBDir, '_parent')) 

989 

990 def tearDown(self): 

991 if os.path.exists(self.testDir): 

992 shutil.rmtree(self.testDir, True) 

993 

994 def test(self): 

995 """Verify that the tags on a repository with an Old Butler repository parent are applied to that 

996 parent 

997 """ 

998 # put objA in repoA: 

999 objA = tstObj('a') 

1000 butler = dp.Butler(outputs=self.repoADir) 

1001 butler.put(objA, 'foo', {'bar': 1}) 

1002 del butler 

1003 

1004 # create repoB and put objB in it: 

1005 objB = tstObj('b') 

1006 butler = dp.Butler(inputs=self.repoADir, outputs=self.repoBDir) 

1007 butler.put(objB, 'foo', {'bar': 2}) 

1008 del butler 

1009 

1010 # verify that repoB can be used as an input, and objA and objB can be gotten from it: 

1011 butler = dp.Butler(inputs=self.repoBDir) 

1012 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA) 

1013 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB) 

1014 del butler 

1015 

1016 # apply a tag and verify that repoB can still be used as an input, and both objA (in repoA) and objB 

1017 # can be gotten from it: 

1018 butler = dp.Butler(inputs={'root': self.repoBDir, 'tags': 'baz'}) 

1019 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA) 

1020 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB) 

1021 del butler 

1022 

1023 # create a New Butler repoC and put objC in it: 

1024 objC = tstObj('c') 

1025 repoCDir = os.path.join(self.testDir, 'repoC') 

1026 butler = dp.Butler(inputs=self.repoBDir, outputs=repoCDir) 

1027 butler.put(objC, 'foo', {'bar': 3}) 

1028 del butler 

1029 

1030 # verify that repoC can be used as an input, and objA, objB, and objC can be gotten from it: 

1031 butler = dp.Butler(inputs=repoCDir) 

1032 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1})), objA) 

1033 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2})), objB) 

1034 self.assertEqual(butler.get('foo', dp.DataId({'bar': 3})), objC) 

1035 del butler 

1036 

1037 # apply a tag and verify that repoC can be used as an input, and objA, objB, and objC can be gotten 

1038 # from it: 

1039 butler = dp.Butler(inputs={'root': repoCDir, 'tags': 'baz'}) 

1040 self.assertEqual(butler.get('foo', dp.DataId({'bar': 1}, tag='baz')), objA) 

1041 self.assertEqual(butler.get('foo', dp.DataId({'bar': 2}, tag='baz')), objB) 

1042 self.assertEqual(butler.get('foo', dp.DataId({'bar': 3}, tag='baz')), objC) 

1043 del butler 

1044 

1045 

1046class MemoryTester(lsst.utils.tests.MemoryTestCase): 

1047 pass 

1048 

1049 

1050if __name__ == '__main__': 1050 ↛ 1051line 1050 didn't jump to line 1051, because the condition on line 1050 was never true

1051 lsst.utils.tests.init() 

1052 unittest.main()