Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import shutil 

25import yaml 

26import tempfile 

27import lsst.utils.tests 

28 

29from lsst.utils import doImport 

30 

31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset 

32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError 

33from lsst.daf.butler.formatters.yaml import YamlFormatter 

34 

35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter, 

36 BadNoWriteFormatter, MetricsExample, DummyRegistry) 

37 

38 

39TESTDIR = os.path.dirname(__file__) 

40 

41 

42def makeExampleMetrics(use_none=False): 

43 if use_none: 

44 array = None 

45 else: 

46 array = [563, 234, 456.7, 105, 2054, -1045] 

47 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

48 {"a": [1, 2, 3], 

49 "b": {"blue": 5, "red": "green"}}, 

50 array, 

51 ) 

52 

53 

54class TransactionTestError(Exception): 

55 """Specific error for transactions, to prevent misdiagnosing 

56 that might otherwise occur when a standard exception is used. 

57 """ 

58 pass 

59 

60 

61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper): 

62 """Support routines for datastore testing""" 

63 root = None 

64 

65 @classmethod 

66 def setUpClass(cls): 

67 # Storage Classes are fixed for all datastores in these tests 

68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") 

69 cls.storageClassFactory = StorageClassFactory() 

70 cls.storageClassFactory.addFromConfig(scConfigFile) 

71 

72 # Read the Datastore config so we can get the class 

73 # information (since we should not assume the constructor 

74 # name here, but rely on the configuration file itself) 

75 datastoreConfig = DatastoreConfig(cls.configFile) 

76 cls.datastoreType = doImport(datastoreConfig["cls"]) 

77 cls.universe = DimensionUniverse() 

78 

79 def setUp(self): 

80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig) 

81 

82 def tearDown(self): 

83 if self.root is not None and os.path.exists(self.root): 

84 shutil.rmtree(self.root, ignore_errors=True) 

85 

86 

87class DatastoreTests(DatastoreTestsBase): 

88 """Some basic tests of a simple datastore.""" 

89 

90 hasUnsupportedPut = True 

91 

92 def testConfigRoot(self): 

93 full = DatastoreConfig(self.configFile) 

94 config = DatastoreConfig(self.configFile, mergeDefaults=False) 

95 newroot = "/random/location" 

96 self.datastoreType.setConfigRoot(newroot, config, full) 

97 if self.rootKeys: 

98 for k in self.rootKeys: 

99 self.assertIn(newroot, config[k]) 

100 

101 def testConstructor(self): 

102 datastore = self.makeDatastore() 

103 self.assertIsNotNone(datastore) 

104 self.assertIs(datastore.isEphemeral, self.isEphemeral) 

105 

106 def testConfigurationValidation(self): 

107 datastore = self.makeDatastore() 

108 sc = self.storageClassFactory.getStorageClass("ThingOne") 

109 datastore.validateConfiguration([sc]) 

110 

111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo") 

112 if self.validationCanFail: 

113 with self.assertRaises(DatastoreValidationError): 

114 datastore.validateConfiguration([sc2], logFailures=True) 

115 

116 dimensions = self.universe.extract(("visit", "physical_filter")) 

117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

119 datastore.validateConfiguration([ref]) 

120 

121 def testParameterValidation(self): 

122 """Check that parameters are validated""" 

123 sc = self.storageClassFactory.getStorageClass("ThingOne") 

124 dimensions = self.universe.extract(("visit", "physical_filter")) 

125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

127 datastore = self.makeDatastore() 

128 data = {1: 2, 3: 4} 

129 datastore.put(data, ref) 

130 newdata = datastore.get(ref) 

131 self.assertEqual(data, newdata) 

132 with self.assertRaises(KeyError): 

133 newdata = datastore.get(ref, parameters={"missing": 5}) 

134 

135 def testBasicPutGet(self): 

136 metrics = makeExampleMetrics() 

137 datastore = self.makeDatastore() 

138 

139 # Create multiple storage classes for testing different formulations 

140 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

141 for sc in ("StructuredData", 

142 "StructuredDataJson", 

143 "StructuredDataPickle")] 

144 

145 dimensions = self.universe.extract(("visit", "physical_filter")) 

146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

147 

148 for sc in storageClasses: 

149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

150 print("Using storageClass: {}".format(sc.name)) 

151 datastore.put(metrics, ref) 

152 

153 # Does it exist? 

154 self.assertTrue(datastore.exists(ref)) 

155 

156 # Get 

157 metricsOut = datastore.get(ref, parameters=None) 

158 self.assertEqual(metrics, metricsOut) 

159 

160 uri = datastore.getURI(ref) 

161 self.assertEqual(uri.scheme, self.uriScheme) 

162 

163 # Get a component -- we need to construct new refs for them 

164 # with derived storage classes but with parent ID 

165 for comp in ("data", "output"): 

166 compRef = ref.makeComponentRef(comp) 

167 output = datastore.get(compRef) 

168 self.assertEqual(output, getattr(metricsOut, comp)) 

169 

170 uri = datastore.getURI(compRef) 

171 self.assertEqual(uri.scheme, self.uriScheme) 

172 

173 storageClass = sc 

174 

175 # Check that we can put a metric with None in a component and 

176 # get it back as None 

177 metricsNone = makeExampleMetrics(use_none=True) 

178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"} 

179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False) 

180 datastore.put(metricsNone, refNone) 

181 

182 comp = "data" 

183 for comp in ("data", "output"): 

184 compRef = refNone.makeComponentRef(comp) 

185 output = datastore.get(compRef) 

186 self.assertEqual(output, getattr(metricsNone, comp)) 

187 

188 # Check that a put fails if the dataset type is not supported 

189 if self.hasUnsupportedPut: 

190 sc = StorageClass("UnsupportedSC", pytype=type(metrics)) 

191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId) 

192 with self.assertRaises(DatasetTypeNotSupportedError): 

193 datastore.put(metrics, ref) 

194 

195 # These should raise 

196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000) 

197 with self.assertRaises(FileNotFoundError): 

198 # non-existing file 

199 datastore.get(ref) 

200 

201 # Get a URI from it 

202 uri = datastore.getURI(ref, predict=True) 

203 self.assertEqual(uri.scheme, self.uriScheme) 

204 

205 with self.assertRaises(FileNotFoundError): 

206 datastore.getURI(ref) 

207 

208 def testDisassembly(self): 

209 """Test disassembly within datastore.""" 

210 metrics = makeExampleMetrics() 

211 if self.isEphemeral: 

212 # in-memory datastore does not disassemble 

213 return 

214 

215 # Create multiple storage classes for testing different formulations 

216 # of composites. One of these will not disassemble to provide 

217 # a reference. 

218 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

219 for sc in ("StructuredComposite", 

220 "StructuredCompositeTestA", 

221 "StructuredCompositeTestB", 

222 "StructuredCompositeReadComp", 

223 "StructuredData", # No disassembly 

224 "StructuredCompositeReadCompNoDisassembly", 

225 )] 

226 

227 # Create the test datastore 

228 datastore = self.makeDatastore() 

229 

230 # Dummy dataId 

231 dimensions = self.universe.extract(("visit", "physical_filter")) 

232 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

233 

234 for i, sc in enumerate(storageClasses): 

235 with self.subTest(storageClass=sc.name): 

236 # Create a different dataset type each time round 

237 # so that a test failure in this subtest does not trigger 

238 # a cascade of tests because of file clashes 

239 ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId, 

240 conform=False) 

241 

242 disassembled = sc.name not in {"StructuredData", "StructuredCompositeReadCompNoDisassembly"} 

243 

244 datastore.put(metrics, ref) 

245 

246 baseURI, compURIs = datastore.getURIs(ref) 

247 if disassembled: 

248 self.assertIsNone(baseURI) 

249 self.assertEqual(set(compURIs), {"data", "output", "summary"}) 

250 else: 

251 self.assertIsNotNone(baseURI) 

252 self.assertEqual(compURIs, {}) 

253 

254 metrics_get = datastore.get(ref) 

255 self.assertEqual(metrics_get, metrics) 

256 

257 # Retrieve the composite with read parameter 

258 stop = 4 

259 metrics_get = datastore.get(ref, parameters={"slice": slice(stop)}) 

260 self.assertEqual(metrics_get.summary, metrics.summary) 

261 self.assertEqual(metrics_get.output, metrics.output) 

262 self.assertEqual(metrics_get.data, metrics.data[:stop]) 

263 

264 # Retrieve a component 

265 data = datastore.get(ref.makeComponentRef("data")) 

266 self.assertEqual(data, metrics.data) 

267 

268 # On supported storage classes attempt to access a read 

269 # only component 

270 if "ReadComp" in sc.name: 

271 cRef = ref.makeComponentRef("counter") 

272 counter = datastore.get(cRef) 

273 self.assertEqual(counter, len(metrics.data)) 

274 

275 counter = datastore.get(cRef, parameters={"slice": slice(stop)}) 

276 self.assertEqual(counter, stop) 

277 

278 datastore.remove(ref) 

279 

280 def testRegistryCompositePutGet(self): 

281 """Tests the case where registry disassembles and puts to datastore. 

282 """ 

283 metrics = makeExampleMetrics() 

284 datastore = self.makeDatastore() 

285 

286 # Create multiple storage classes for testing different formulations 

287 # of composites 

288 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

289 for sc in ("StructuredComposite", 

290 "StructuredCompositeTestA", 

291 "StructuredCompositeTestB", 

292 )] 

293 

294 dimensions = self.universe.extract(("visit", "physical_filter")) 

295 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

296 

297 for sc in storageClasses: 

298 print("Using storageClass: {}".format(sc.name)) 

299 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, 

300 conform=False) 

301 

302 components = sc.assembler().disassemble(metrics) 

303 self.assertTrue(components) 

304 

305 compsRead = {} 

306 for compName, compInfo in components.items(): 

307 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

308 components[compName].storageClass, dataId, 

309 conform=False) 

310 

311 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name)) 

312 datastore.put(compInfo.component, compRef) 

313 

314 uri = datastore.getURI(compRef) 

315 self.assertEqual(uri.scheme, self.uriScheme) 

316 

317 compsRead[compName] = datastore.get(compRef) 

318 

319 # We can generate identical files for each storage class 

320 # so remove the component here 

321 datastore.remove(compRef) 

322 

323 # combine all the components we read back into a new composite 

324 metricsOut = sc.assembler().assemble(compsRead) 

325 self.assertEqual(metrics, metricsOut) 

326 

327 def testRemove(self): 

328 metrics = makeExampleMetrics() 

329 datastore = self.makeDatastore() 

330 # Put 

331 dimensions = self.universe.extract(("visit", "physical_filter")) 

332 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"} 

333 

334 sc = self.storageClassFactory.getStorageClass("StructuredData") 

335 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

336 datastore.put(metrics, ref) 

337 

338 # Does it exist? 

339 self.assertTrue(datastore.exists(ref)) 

340 

341 # Get 

342 metricsOut = datastore.get(ref) 

343 self.assertEqual(metrics, metricsOut) 

344 # Remove 

345 datastore.remove(ref) 

346 

347 # Does it exist? 

348 self.assertFalse(datastore.exists(ref)) 

349 

350 # Do we now get a predicted URI? 

351 uri = datastore.getURI(ref, predict=True) 

352 self.assertEqual(uri.fragment, "predicted") 

353 

354 # Get should now fail 

355 with self.assertRaises(FileNotFoundError): 

356 datastore.get(ref) 

357 # Can only delete once 

358 with self.assertRaises(FileNotFoundError): 

359 datastore.remove(ref) 

360 

361 def testTransfer(self): 

362 metrics = makeExampleMetrics() 

363 

364 dimensions = self.universe.extract(("visit", "physical_filter")) 

365 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"} 

366 

367 sc = self.storageClassFactory.getStorageClass("StructuredData") 

368 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

369 

370 inputDatastore = self.makeDatastore("test_input_datastore") 

371 outputDatastore = self.makeDatastore("test_output_datastore") 

372 

373 inputDatastore.put(metrics, ref) 

374 outputDatastore.transfer(inputDatastore, ref) 

375 

376 metricsOut = outputDatastore.get(ref) 

377 self.assertEqual(metrics, metricsOut) 

378 

379 def testBasicTransaction(self): 

380 datastore = self.makeDatastore() 

381 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

382 dimensions = self.universe.extract(("visit", "physical_filter")) 

383 nDatasets = 6 

384 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)] 

385 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False), 

386 makeExampleMetrics(),) 

387 for dataId in dataIds] 

388 succeed = data[:nDatasets//2] 

389 fail = data[nDatasets//2:] 

390 # All datasets added in this transaction should continue to exist 

391 with datastore.transaction(): 

392 for ref, metrics in succeed: 

393 datastore.put(metrics, ref) 

394 # Whereas datasets added in this transaction should not 

395 with self.assertRaises(TransactionTestError): 

396 with datastore.transaction(): 

397 for ref, metrics in fail: 

398 datastore.put(metrics, ref) 

399 raise TransactionTestError("This should propagate out of the context manager") 

400 # Check for datasets that should exist 

401 for ref, metrics in succeed: 

402 # Does it exist? 

403 self.assertTrue(datastore.exists(ref)) 

404 # Get 

405 metricsOut = datastore.get(ref, parameters=None) 

406 self.assertEqual(metrics, metricsOut) 

407 # URI 

408 uri = datastore.getURI(ref) 

409 self.assertEqual(uri.scheme, self.uriScheme) 

410 # Check for datasets that should not exist 

411 for ref, _ in fail: 

412 # These should raise 

413 with self.assertRaises(FileNotFoundError): 

414 # non-existing file 

415 datastore.get(ref) 

416 with self.assertRaises(FileNotFoundError): 

417 datastore.getURI(ref) 

418 

419 def testNestedTransaction(self): 

420 datastore = self.makeDatastore() 

421 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

422 dimensions = self.universe.extract(("visit", "physical_filter")) 

423 metrics = makeExampleMetrics() 

424 

425 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

426 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

427 conform=False) 

428 datastore.put(metrics, refBefore) 

429 with self.assertRaises(TransactionTestError): 

430 with datastore.transaction(): 

431 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"} 

432 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

433 conform=False) 

434 datastore.put(metrics, refOuter) 

435 with datastore.transaction(): 

436 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"} 

437 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

438 conform=False) 

439 datastore.put(metrics, refInner) 

440 # All datasets should exist 

441 for ref in (refBefore, refOuter, refInner): 

442 metricsOut = datastore.get(ref, parameters=None) 

443 self.assertEqual(metrics, metricsOut) 

444 raise TransactionTestError("This should roll back the transaction") 

445 # Dataset(s) inserted before the transaction should still exist 

446 metricsOut = datastore.get(refBefore, parameters=None) 

447 self.assertEqual(metrics, metricsOut) 

448 # But all datasets inserted during the (rolled back) transaction 

449 # should be gone 

450 with self.assertRaises(FileNotFoundError): 

451 datastore.get(refOuter) 

452 with self.assertRaises(FileNotFoundError): 

453 datastore.get(refInner) 

454 

455 def _prepareIngestTest(self): 

456 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

457 dimensions = self.universe.extract(("visit", "physical_filter")) 

458 metrics = makeExampleMetrics() 

459 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

460 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

461 return metrics, ref 

462 

463 def runIngestTest(self, func, expectOutput=True): 

464 metrics, ref = self._prepareIngestTest() 

465 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path: 

466 with open(path, 'w') as fd: 

467 yaml.dump(metrics._asdict(), stream=fd) 

468 func(metrics, path, ref) 

469 

470 def testIngestNoTransfer(self): 

471 """Test ingesting existing files with no transfer. 

472 """ 

473 for mode in (None, "auto"): 

474 

475 # Some datastores have auto but can't do in place transfer 

476 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto: 

477 continue 

478 

479 with self.subTest(mode=mode): 

480 datastore = self.makeDatastore() 

481 

482 def succeed(obj, path, ref): 

483 """Ingest a file already in the datastore root.""" 

484 # first move it into the root, and adjust the path 

485 # accordingly 

486 path = shutil.copy(path, datastore.root) 

487 path = os.path.relpath(path, start=datastore.root) 

488 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

489 self.assertEqual(obj, datastore.get(ref)) 

490 

491 def failInputDoesNotExist(obj, path, ref): 

492 """Can't ingest files if we're given a bad path.""" 

493 with self.assertRaises(FileNotFoundError): 

494 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref), 

495 transfer=mode) 

496 self.assertFalse(datastore.exists(ref)) 

497 

498 def failOutsideRoot(obj, path, ref): 

499 """Can't ingest files outside of datastore root unless 

500 auto.""" 

501 if mode == "auto": 

502 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

503 self.assertTrue(datastore.exists(ref)) 

504 else: 

505 with self.assertRaises(RuntimeError): 

506 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

507 self.assertFalse(datastore.exists(ref)) 

508 

509 def failNotImplemented(obj, path, ref): 

510 with self.assertRaises(NotImplementedError): 

511 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

512 

513 if mode in self.ingestTransferModes: 

514 self.runIngestTest(failOutsideRoot) 

515 self.runIngestTest(failInputDoesNotExist) 

516 self.runIngestTest(succeed) 

517 else: 

518 self.runIngestTest(failNotImplemented) 

519 

520 def testIngestTransfer(self): 

521 """Test ingesting existing files after transferring them. 

522 """ 

523 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"): 

524 with self.subTest(mode=mode): 

525 datastore = self.makeDatastore(mode) 

526 

527 def succeed(obj, path, ref): 

528 """Ingest a file by transferring it to the template 

529 location.""" 

530 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

531 self.assertEqual(obj, datastore.get(ref)) 

532 

533 def failInputDoesNotExist(obj, path, ref): 

534 """Can't ingest files if we're given a bad path.""" 

535 with self.assertRaises(FileNotFoundError): 

536 # Ensure the file does not look like it is in 

537 # datastore for auto mode 

538 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref), 

539 transfer=mode) 

540 self.assertFalse(datastore.exists(ref)) 

541 

542 def failOutputExists(obj, path, ref): 

543 """Can't ingest files if transfer destination already 

544 exists.""" 

545 with self.assertRaises(FileExistsError): 

546 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

547 self.assertFalse(datastore.exists(ref)) 

548 

549 def failNotImplemented(obj, path, ref): 

550 with self.assertRaises(NotImplementedError): 

551 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

552 

553 if mode in self.ingestTransferModes: 

554 self.runIngestTest(failInputDoesNotExist) 

555 self.runIngestTest(succeed, expectOutput=(mode != "move")) 

556 self.runIngestTest(failOutputExists) 

557 else: 

558 self.runIngestTest(failNotImplemented) 

559 

560 def testIngestSymlinkOfSymlink(self): 

561 """Special test for symlink to a symlink ingest""" 

562 metrics, ref = self._prepareIngestTest() 

563 # The aim of this test is to create a dataset on disk, then 

564 # create a symlink to it and finally ingest the symlink such that 

565 # the symlink in the datastore points to the original dataset. 

566 for mode in ("symlink", "relsymlink"): 

567 if mode not in self.ingestTransferModes: 

568 continue 

569 

570 print(f"Trying mode {mode}") 

571 with lsst.utils.tests.getTempFilePath(".yaml") as realpath: 

572 with open(realpath, 'w') as fd: 

573 yaml.dump(metrics._asdict(), stream=fd) 

574 with lsst.utils.tests.getTempFilePath(".yaml") as sympath: 

575 os.symlink(os.path.abspath(realpath), sympath) 

576 

577 datastore = self.makeDatastore() 

578 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode) 

579 

580 uri = datastore.getURI(ref) 

581 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}") 

582 self.assertTrue(os.path.islink(uri.path)) 

583 

584 linkTarget = os.readlink(uri.path) 

585 if mode == "relsymlink": 

586 self.assertFalse(os.path.isabs(linkTarget)) 

587 else: 

588 self.assertEqual(linkTarget, os.path.abspath(realpath)) 

589 

590 # Check that we can get the dataset back regardless of mode 

591 metric2 = datastore.get(ref) 

592 self.assertEqual(metric2, metrics) 

593 

594 # Cleanup the file for next time round loop 

595 # since it will get the same file name in store 

596 datastore.remove(ref) 

597 

598 

599class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase): 

600 """PosixDatastore specialization""" 

601 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

602 uriScheme = "file" 

603 canIngestNoTransferAuto = True 

604 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto") 

605 isEphemeral = False 

606 rootKeys = ("root",) 

607 validationCanFail = True 

608 

609 def setUp(self): 

610 # Override the working directory before calling the base class 

611 self.root = tempfile.mkdtemp(dir=TESTDIR) 

612 super().setUp() 

613 

614 

615class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase): 

616 """Posix datastore tests but with checksums disabled.""" 

617 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml") 

618 

619 def testChecksum(self): 

620 """Ensure that checksums have not been calculated.""" 

621 

622 datastore = self.makeDatastore() 

623 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

624 dimensions = self.universe.extract(("visit", "physical_filter")) 

625 metrics = makeExampleMetrics() 

626 

627 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

628 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

629 conform=False) 

630 

631 # Configuration should have disabled checksum calculation 

632 datastore.put(metrics, ref) 

633 infos = datastore.getStoredItemsInfo(ref) 

634 self.assertIsNone(infos[0].checksum) 

635 

636 # Remove put back but with checksums enabled explicitly 

637 datastore.remove(ref) 

638 datastore.useChecksum = True 

639 datastore.put(metrics, ref) 

640 

641 infos = datastore.getStoredItemsInfo(ref) 

642 self.assertIsNotNone(infos[0].checksum) 

643 

644 

645class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase): 

646 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

647 

648 def setUp(self): 

649 # Override the working directory before calling the base class 

650 self.root = tempfile.mkdtemp(dir=TESTDIR) 

651 super().setUp() 

652 

653 def testCleanup(self): 

654 """Test that a failed formatter write does cleanup a partial file.""" 

655 metrics = makeExampleMetrics() 

656 datastore = self.makeDatastore() 

657 

658 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

659 

660 dimensions = self.universe.extract(("visit", "physical_filter")) 

661 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

662 

663 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

664 

665 # Determine where the file will end up (we assume Formatters use 

666 # the same file extension) 

667 expectedUri = datastore.getURI(ref, predict=True) 

668 self.assertEqual(expectedUri.fragment, "predicted") 

669 

670 expectedFile = expectedUri.path 

671 self.assertTrue(expectedFile.endswith(".yaml"), 

672 f"Is there a file extension in {expectedUri}") 

673 

674 # Try formatter that fails and formatter that fails and leaves 

675 # a file behind 

676 for formatter in (BadWriteFormatter, BadNoWriteFormatter): 

677 with self.subTest(formatter=formatter): 

678 

679 # Monkey patch the formatter 

680 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter, 

681 overwrite=True) 

682 

683 # Try to put the dataset, it should fail 

684 with self.assertRaises(Exception): 

685 datastore.put(metrics, ref) 

686 

687 # Check that there is no file on disk 

688 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

689 

690 # Check that there is a directory 

691 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)), 

692 f"Check for existence of directory {os.path.dirname(expectedFile)}") 

693 

694 # Force YamlFormatter and check that this time a file is written 

695 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter, 

696 overwrite=True) 

697 datastore.put(metrics, ref) 

698 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

699 datastore.remove(ref) 

700 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}") 

701 

702 

703class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase): 

704 """PosixDatastore specialization""" 

705 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml") 

706 uriScheme = "mem" 

707 hasUnsupportedPut = False 

708 ingestTransferModes = () 

709 isEphemeral = True 

710 rootKeys = None 

711 validationCanFail = False 

712 

713 

714class ChainedDatastoreTestCase(PosixDatastoreTestCase): 

715 """ChainedDatastore specialization using a POSIXDatastore""" 

716 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml") 

717 hasUnsupportedPut = False 

718 canIngestNoTransferAuto = False 

719 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto") 

720 isEphemeral = False 

721 rootKeys = (".datastores.1.root", ".datastores.2.root") 

722 validationCanFail = True 

723 

724 

725class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase): 

726 """ChainedDatastore specialization using all InMemoryDatastore""" 

727 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml") 

728 validationCanFail = False 

729 

730 

731class DatastoreConstraintsTests(DatastoreTestsBase): 

732 """Basic tests of constraints model of Datastores.""" 

733 

734 def testConstraints(self): 

735 """Test constraints model. Assumes that each test class has the 

736 same constraints.""" 

737 metrics = makeExampleMetrics() 

738 datastore = self.makeDatastore() 

739 

740 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

741 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

742 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

743 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

744 

745 # Write empty file suitable for ingest check (JSON and YAML variants) 

746 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

747 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

748 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False), 

749 ("metric33", sc1, True), ("metric2", sc2, True)): 

750 # Choose different temp file depending on StorageClass 

751 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

752 

753 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name): 

754 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False) 

755 if accepted: 

756 datastore.put(metrics, ref) 

757 self.assertTrue(datastore.exists(ref)) 

758 datastore.remove(ref) 

759 

760 # Try ingest 

761 if self.canIngest: 

762 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

763 self.assertTrue(datastore.exists(ref)) 

764 datastore.remove(ref) 

765 else: 

766 with self.assertRaises(DatasetTypeNotSupportedError): 

767 datastore.put(metrics, ref) 

768 self.assertFalse(datastore.exists(ref)) 

769 

770 # Again with ingest 

771 if self.canIngest: 

772 with self.assertRaises(DatasetTypeNotSupportedError): 

773 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

774 self.assertFalse(datastore.exists(ref)) 

775 

776 

777class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

778 """PosixDatastore specialization""" 

779 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml") 

780 canIngest = True 

781 

782 def setUp(self): 

783 # Override the working directory before calling the base class 

784 self.root = tempfile.mkdtemp(dir=TESTDIR) 

785 super().setUp() 

786 

787 

788class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

789 """InMemoryDatastore specialization""" 

790 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml") 

791 canIngest = False 

792 

793 

794class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase): 

795 """ChainedDatastore specialization using a POSIXDatastore and constraints 

796 at the ChainedDatstore """ 

797 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml") 

798 

799 

800class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase): 

801 """ChainedDatastore specialization using a POSIXDatastore""" 

802 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml") 

803 

804 

805class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase): 

806 """ChainedDatastore specialization using all InMemoryDatastore""" 

807 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml") 

808 canIngest = False 

809 

810 

811class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase): 

812 """Test that a chained datastore can control constraints per-datastore 

813 even if child datastore would accept.""" 

814 

815 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml") 

816 

817 def setUp(self): 

818 # Override the working directory before calling the base class 

819 self.root = tempfile.mkdtemp(dir=TESTDIR) 

820 super().setUp() 

821 

822 def testConstraints(self): 

823 """Test chained datastore constraints model.""" 

824 metrics = makeExampleMetrics() 

825 datastore = self.makeDatastore() 

826 

827 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

828 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

829 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

830 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

831 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"} 

832 

833 # Write empty file suitable for ingest check (JSON and YAML variants) 

834 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

835 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

836 

837 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True), 

838 ("metric2", dataId1, sc1, (False, False, False), False), 

839 ("metric2", dataId2, sc1, (True, False, False), False), 

840 ("metric33", dataId2, sc2, (True, True, False), True), 

841 ("metric2", dataId1, sc2, (False, True, False), True)): 

842 

843 # Choose different temp file depending on StorageClass 

844 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

845 

846 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name): 

847 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId, 

848 conform=False) 

849 if any(accept): 

850 datastore.put(metrics, ref) 

851 self.assertTrue(datastore.exists(ref)) 

852 

853 # Check each datastore inside the chained datastore 

854 for childDatastore, expected in zip(datastore.datastores, accept): 

855 self.assertEqual(childDatastore.exists(ref), expected, 

856 f"Testing presence of {ref} in datastore {childDatastore.name}") 

857 

858 datastore.remove(ref) 

859 

860 # Check that ingest works 

861 if ingest: 

862 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

863 self.assertTrue(datastore.exists(ref)) 

864 

865 # Check each datastore inside the chained datastore 

866 for childDatastore, expected in zip(datastore.datastores, accept): 

867 # Ephemeral datastores means InMemory at the moment 

868 # and that does not accept ingest of files. 

869 if childDatastore.isEphemeral: 

870 expected = False 

871 self.assertEqual(childDatastore.exists(ref), expected, 

872 f"Testing presence of ingested {ref} in datastore" 

873 f" {childDatastore.name}") 

874 

875 datastore.remove(ref) 

876 else: 

877 with self.assertRaises(DatasetTypeNotSupportedError): 

878 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

879 

880 else: 

881 with self.assertRaises(DatasetTypeNotSupportedError): 

882 datastore.put(metrics, ref) 

883 self.assertFalse(datastore.exists(ref)) 

884 

885 # Again with ingest 

886 with self.assertRaises(DatasetTypeNotSupportedError): 

887 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

888 self.assertFalse(datastore.exists(ref)) 

889 

890 

891if __name__ == "__main__": 891 ↛ 892line 891 didn't jump to line 892, because the condition on line 891 was never true

892 unittest.main()