Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import shutil 

25import yaml 

26import tempfile 

27import lsst.utils 

28 

29from lsst.utils import doImport 

30 

31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset 

32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError 

33from lsst.daf.butler.formatters.yamlFormatter import YamlFormatter 

34 

35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter, 

36 BadNoWriteFormatter, MetricsExample, DummyRegistry) 

37 

38 

39TESTDIR = os.path.dirname(__file__) 

40 

41 

42def makeExampleMetrics(use_none=False): 

43 if use_none: 

44 array = None 

45 else: 

46 array = [563, 234, 456.7] 

47 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

48 {"a": [1, 2, 3], 

49 "b": {"blue": 5, "red": "green"}}, 

50 array, 

51 ) 

52 

53 

54class TransactionTestError(Exception): 

55 """Specific error for transactions, to prevent misdiagnosing 

56 that might otherwise occur when a standard exception is used. 

57 """ 

58 pass 

59 

60 

61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper): 

62 """Support routines for datastore testing""" 

63 root = None 

64 

65 @classmethod 

66 def setUpClass(cls): 

67 # Storage Classes are fixed for all datastores in these tests 

68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") 

69 cls.storageClassFactory = StorageClassFactory() 

70 cls.storageClassFactory.addFromConfig(scConfigFile) 

71 

72 # Read the Datastore config so we can get the class 

73 # information (since we should not assume the constructor 

74 # name here, but rely on the configuration file itself) 

75 datastoreConfig = DatastoreConfig(cls.configFile) 

76 cls.datastoreType = doImport(datastoreConfig["cls"]) 

77 cls.universe = DimensionUniverse() 

78 

79 def setUp(self): 

80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig) 

81 

82 def tearDown(self): 

83 if self.root is not None and os.path.exists(self.root): 

84 shutil.rmtree(self.root, ignore_errors=True) 

85 

86 

87class DatastoreTests(DatastoreTestsBase): 

88 """Some basic tests of a simple datastore.""" 

89 

90 hasUnsupportedPut = True 

91 

92 def testConfigRoot(self): 

93 full = DatastoreConfig(self.configFile) 

94 config = DatastoreConfig(self.configFile, mergeDefaults=False) 

95 newroot = "/random/location" 

96 self.datastoreType.setConfigRoot(newroot, config, full) 

97 if self.rootKeys: 

98 for k in self.rootKeys: 

99 self.assertIn(newroot, config[k]) 

100 

101 def testConstructor(self): 

102 datastore = self.makeDatastore() 

103 self.assertIsNotNone(datastore) 

104 self.assertIs(datastore.isEphemeral, self.isEphemeral) 

105 

106 def testConfigurationValidation(self): 

107 datastore = self.makeDatastore() 

108 sc = self.storageClassFactory.getStorageClass("ThingOne") 

109 datastore.validateConfiguration([sc]) 

110 

111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo") 

112 if self.validationCanFail: 

113 with self.assertRaises(DatastoreValidationError): 

114 datastore.validateConfiguration([sc2], logFailures=True) 

115 

116 dimensions = self.universe.extract(("visit", "physical_filter")) 

117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

119 datastore.validateConfiguration([ref]) 

120 

121 def testParameterValidation(self): 

122 """Check that parameters are validated""" 

123 sc = self.storageClassFactory.getStorageClass("ThingOne") 

124 dimensions = self.universe.extract(("visit", "physical_filter")) 

125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

127 datastore = self.makeDatastore() 

128 data = {1: 2, 3: 4} 

129 datastore.put(data, ref) 

130 newdata = datastore.get(ref) 

131 self.assertEqual(data, newdata) 

132 with self.assertRaises(KeyError): 

133 newdata = datastore.get(ref, parameters={"missing": 5}) 

134 

135 def testBasicPutGet(self): 

136 metrics = makeExampleMetrics() 

137 datastore = self.makeDatastore() 

138 

139 # Create multiple storage classes for testing different formulations 

140 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

141 for sc in ("StructuredData", 

142 "StructuredDataJson", 

143 "StructuredDataPickle")] 

144 

145 dimensions = self.universe.extract(("visit", "physical_filter")) 

146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

147 

148 for sc in storageClasses: 

149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

150 print("Using storageClass: {}".format(sc.name)) 

151 datastore.put(metrics, ref) 

152 

153 # Does it exist? 

154 self.assertTrue(datastore.exists(ref)) 

155 

156 # Get 

157 metricsOut = datastore.get(ref, parameters=None) 

158 self.assertEqual(metrics, metricsOut) 

159 

160 uri = datastore.getURI(ref) 

161 self.assertEqual(uri.scheme, self.uriScheme) 

162 

163 # Get a component -- we need to construct new refs for them 

164 # with derived storage classes but with parent ID 

165 for comp in ("data", "output"): 

166 compRef = ref.makeComponentRef(comp) 

167 output = datastore.get(compRef) 

168 self.assertEqual(output, getattr(metricsOut, comp)) 

169 

170 uri = datastore.getURI(compRef) 

171 self.assertEqual(uri.scheme, self.uriScheme) 

172 

173 storageClass = sc 

174 

175 # Check that we can put a metric with None in a component and 

176 # get it back as None 

177 metricsNone = makeExampleMetrics(use_none=True) 

178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"} 

179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False) 

180 datastore.put(metricsNone, refNone) 

181 

182 comp = "data" 

183 for comp in ("data", "output"): 

184 compRef = refNone.makeComponentRef(comp) 

185 output = datastore.get(compRef) 

186 self.assertEqual(output, getattr(metricsNone, comp)) 

187 

188 # Check that a put fails if the dataset type is not supported 

189 if self.hasUnsupportedPut: 

190 sc = StorageClass("UnsupportedSC", pytype=type(metrics)) 

191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId) 

192 with self.assertRaises(DatasetTypeNotSupportedError): 

193 datastore.put(metrics, ref) 

194 

195 # These should raise 

196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000) 

197 with self.assertRaises(FileNotFoundError): 

198 # non-existing file 

199 datastore.get(ref) 

200 

201 # Get a URI from it 

202 uri = datastore.getURI(ref, predict=True) 

203 self.assertEqual(uri.scheme, self.uriScheme) 

204 

205 with self.assertRaises(FileNotFoundError): 

206 datastore.getURI(ref) 

207 

208 def testCompositePutGet(self): 

209 metrics = makeExampleMetrics() 

210 datastore = self.makeDatastore() 

211 

212 # Create multiple storage classes for testing different formulations 

213 # of composites 

214 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

215 for sc in ("StructuredComposite", 

216 "StructuredCompositeTestA", 

217 "StructuredCompositeTestB")] 

218 

219 dimensions = self.universe.extract(("visit", "physical_filter")) 

220 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

221 

222 for sc in storageClasses: 

223 print("Using storageClass: {}".format(sc.name)) 

224 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, 

225 conform=False) 

226 

227 components = sc.assembler().disassemble(metrics) 

228 self.assertTrue(components) 

229 

230 compsRead = {} 

231 for compName, compInfo in components.items(): 

232 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

233 components[compName].storageClass, dataId, 

234 conform=False) 

235 

236 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name)) 

237 datastore.put(compInfo.component, compRef) 

238 

239 uri = datastore.getURI(compRef) 

240 self.assertEqual(uri.scheme, self.uriScheme) 

241 

242 compsRead[compName] = datastore.get(compRef) 

243 

244 # We can generate identical files for each storage class 

245 # so remove the component here 

246 datastore.remove(compRef) 

247 

248 # combine all the components we read back into a new composite 

249 metricsOut = sc.assembler().assemble(compsRead) 

250 self.assertEqual(metrics, metricsOut) 

251 

252 def testRemove(self): 

253 metrics = makeExampleMetrics() 

254 datastore = self.makeDatastore() 

255 # Put 

256 dimensions = self.universe.extract(("visit", "physical_filter")) 

257 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"} 

258 

259 sc = self.storageClassFactory.getStorageClass("StructuredData") 

260 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

261 datastore.put(metrics, ref) 

262 

263 # Does it exist? 

264 self.assertTrue(datastore.exists(ref)) 

265 

266 # Get 

267 metricsOut = datastore.get(ref) 

268 self.assertEqual(metrics, metricsOut) 

269 # Remove 

270 datastore.remove(ref) 

271 

272 # Does it exist? 

273 self.assertFalse(datastore.exists(ref)) 

274 

275 # Do we now get a predicted URI? 

276 uri = datastore.getURI(ref, predict=True) 

277 self.assertEqual(uri.fragment, "predicted") 

278 

279 # Get should now fail 

280 with self.assertRaises(FileNotFoundError): 

281 datastore.get(ref) 

282 # Can only delete once 

283 with self.assertRaises(FileNotFoundError): 

284 datastore.remove(ref) 

285 

286 def testTransfer(self): 

287 metrics = makeExampleMetrics() 

288 

289 dimensions = self.universe.extract(("visit", "physical_filter")) 

290 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"} 

291 

292 sc = self.storageClassFactory.getStorageClass("StructuredData") 

293 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

294 

295 inputDatastore = self.makeDatastore("test_input_datastore") 

296 outputDatastore = self.makeDatastore("test_output_datastore") 

297 

298 inputDatastore.put(metrics, ref) 

299 outputDatastore.transfer(inputDatastore, ref) 

300 

301 metricsOut = outputDatastore.get(ref) 

302 self.assertEqual(metrics, metricsOut) 

303 

304 def testBasicTransaction(self): 

305 datastore = self.makeDatastore() 

306 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

307 dimensions = self.universe.extract(("visit", "physical_filter")) 

308 nDatasets = 6 

309 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)] 

310 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False), 

311 makeExampleMetrics(),) 

312 for dataId in dataIds] 

313 succeed = data[:nDatasets//2] 

314 fail = data[nDatasets//2:] 

315 # All datasets added in this transaction should continue to exist 

316 with datastore.transaction(): 

317 for ref, metrics in succeed: 

318 datastore.put(metrics, ref) 

319 # Whereas datasets added in this transaction should not 

320 with self.assertRaises(TransactionTestError): 

321 with datastore.transaction(): 

322 for ref, metrics in fail: 

323 datastore.put(metrics, ref) 

324 raise TransactionTestError("This should propagate out of the context manager") 

325 # Check for datasets that should exist 

326 for ref, metrics in succeed: 

327 # Does it exist? 

328 self.assertTrue(datastore.exists(ref)) 

329 # Get 

330 metricsOut = datastore.get(ref, parameters=None) 

331 self.assertEqual(metrics, metricsOut) 

332 # URI 

333 uri = datastore.getURI(ref) 

334 self.assertEqual(uri.scheme, self.uriScheme) 

335 # Check for datasets that should not exist 

336 for ref, _ in fail: 

337 # These should raise 

338 with self.assertRaises(FileNotFoundError): 

339 # non-existing file 

340 datastore.get(ref) 

341 with self.assertRaises(FileNotFoundError): 

342 datastore.getURI(ref) 

343 

344 def testNestedTransaction(self): 

345 datastore = self.makeDatastore() 

346 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

347 dimensions = self.universe.extract(("visit", "physical_filter")) 

348 metrics = makeExampleMetrics() 

349 

350 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

351 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

352 conform=False) 

353 datastore.put(metrics, refBefore) 

354 with self.assertRaises(TransactionTestError): 

355 with datastore.transaction(): 

356 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"} 

357 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

358 conform=False) 

359 datastore.put(metrics, refOuter) 

360 with datastore.transaction(): 

361 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"} 

362 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

363 conform=False) 

364 datastore.put(metrics, refInner) 

365 # All datasets should exist 

366 for ref in (refBefore, refOuter, refInner): 

367 metricsOut = datastore.get(ref, parameters=None) 

368 self.assertEqual(metrics, metricsOut) 

369 raise TransactionTestError("This should roll back the transaction") 

370 # Dataset(s) inserted before the transaction should still exist 

371 metricsOut = datastore.get(refBefore, parameters=None) 

372 self.assertEqual(metrics, metricsOut) 

373 # But all datasets inserted during the (rolled back) transaction 

374 # should be gone 

375 with self.assertRaises(FileNotFoundError): 

376 datastore.get(refOuter) 

377 with self.assertRaises(FileNotFoundError): 

378 datastore.get(refInner) 

379 

380 def _prepareIngestTest(self): 

381 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

382 dimensions = self.universe.extract(("visit", "physical_filter")) 

383 metrics = makeExampleMetrics() 

384 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

385 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

386 return metrics, ref 

387 

388 def runIngestTest(self, func, expectOutput=True): 

389 metrics, ref = self._prepareIngestTest() 

390 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path: 

391 with open(path, 'w') as fd: 

392 yaml.dump(metrics._asdict(), stream=fd) 

393 func(metrics, path, ref) 

394 

395 def testIngestNoTransfer(self): 

396 """Test ingesting existing files with no transfer. 

397 """ 

398 for mode in (None, "auto"): 

399 

400 # Some datastores have auto but can't do in place transfer 

401 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto: 

402 continue 

403 

404 with self.subTest(mode=mode): 

405 datastore = self.makeDatastore() 

406 

407 def succeed(obj, path, ref): 

408 """Ingest a file already in the datastore root.""" 

409 # first move it into the root, and adjust the path 

410 # accordingly 

411 path = shutil.copy(path, datastore.root) 

412 path = os.path.relpath(path, start=datastore.root) 

413 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

414 self.assertEqual(obj, datastore.get(ref)) 

415 

416 def failInputDoesNotExist(obj, path, ref): 

417 """Can't ingest files if we're given a bad path.""" 

418 with self.assertRaises(FileNotFoundError): 

419 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref), 

420 transfer=mode) 

421 self.assertFalse(datastore.exists(ref)) 

422 

423 def failOutsideRoot(obj, path, ref): 

424 """Can't ingest files outside of datastore root unless 

425 auto.""" 

426 if mode == "auto": 

427 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

428 self.assertTrue(datastore.exists(ref)) 

429 else: 

430 with self.assertRaises(RuntimeError): 

431 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

432 self.assertFalse(datastore.exists(ref)) 

433 

434 def failNotImplemented(obj, path, ref): 

435 with self.assertRaises(NotImplementedError): 

436 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

437 

438 if mode in self.ingestTransferModes: 

439 self.runIngestTest(failOutsideRoot) 

440 self.runIngestTest(failInputDoesNotExist) 

441 self.runIngestTest(succeed) 

442 else: 

443 self.runIngestTest(failNotImplemented) 

444 

445 def testIngestTransfer(self): 

446 """Test ingesting existing files after transferring them. 

447 """ 

448 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"): 

449 with self.subTest(mode=mode): 

450 datastore = self.makeDatastore(mode) 

451 

452 def succeed(obj, path, ref): 

453 """Ingest a file by transferring it to the template 

454 location.""" 

455 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

456 self.assertEqual(obj, datastore.get(ref)) 

457 

458 def failInputDoesNotExist(obj, path, ref): 

459 """Can't ingest files if we're given a bad path.""" 

460 with self.assertRaises(FileNotFoundError): 

461 # Ensure the file does not look like it is in 

462 # datastore for auto mode 

463 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref), 

464 transfer=mode) 

465 self.assertFalse(datastore.exists(ref)) 

466 

467 def failOutputExists(obj, path, ref): 

468 """Can't ingest files if transfer destination already 

469 exists.""" 

470 with self.assertRaises(FileExistsError): 

471 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

472 self.assertFalse(datastore.exists(ref)) 

473 

474 def failNotImplemented(obj, path, ref): 

475 with self.assertRaises(NotImplementedError): 

476 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

477 

478 if mode in self.ingestTransferModes: 

479 self.runIngestTest(failInputDoesNotExist) 

480 self.runIngestTest(succeed, expectOutput=(mode != "move")) 

481 self.runIngestTest(failOutputExists) 

482 else: 

483 self.runIngestTest(failNotImplemented) 

484 

485 def testIngestSymlinkOfSymlink(self): 

486 """Special test for symlink to a symlink ingest""" 

487 metrics, ref = self._prepareIngestTest() 

488 # The aim of this test is to create a dataset on disk, then 

489 # create a symlink to it and finally ingest the symlink such that 

490 # the symlink in the datastore points to the original dataset. 

491 for mode in ("symlink", "relsymlink"): 

492 if mode not in self.ingestTransferModes: 

493 continue 

494 

495 print(f"Trying mode {mode}") 

496 with lsst.utils.tests.getTempFilePath(".yaml") as realpath: 

497 with open(realpath, 'w') as fd: 

498 yaml.dump(metrics._asdict(), stream=fd) 

499 with lsst.utils.tests.getTempFilePath(".yaml") as sympath: 

500 os.symlink(os.path.abspath(realpath), sympath) 

501 

502 datastore = self.makeDatastore() 

503 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode) 

504 

505 uri = datastore.getURI(ref) 

506 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}") 

507 self.assertTrue(os.path.islink(uri.path)) 

508 

509 linkTarget = os.readlink(uri.path) 

510 if mode == "relsymlink": 

511 self.assertFalse(os.path.isabs(linkTarget)) 

512 else: 

513 self.assertEqual(linkTarget, os.path.abspath(realpath)) 

514 

515 # Check that we can get the dataset back regardless of mode 

516 metric2 = datastore.get(ref) 

517 self.assertEqual(metric2, metrics) 

518 

519 # Cleanup the file for next time round loop 

520 # since it will get the same file name in store 

521 datastore.remove(ref) 

522 

523 

524class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase): 

525 """PosixDatastore specialization""" 

526 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

527 uriScheme = "file" 

528 canIngestNoTransferAuto = True 

529 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto") 

530 isEphemeral = False 

531 rootKeys = ("root",) 

532 validationCanFail = True 

533 

534 def setUp(self): 

535 # Override the working directory before calling the base class 

536 self.root = tempfile.mkdtemp(dir=TESTDIR) 

537 super().setUp() 

538 

539 

540class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase): 

541 """Posix datastore tests but with checksums disabled.""" 

542 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml") 

543 

544 def testChecksum(self): 

545 """Ensure that checksums have not been calculated.""" 

546 

547 datastore = self.makeDatastore() 

548 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

549 dimensions = self.universe.extract(("visit", "physical_filter")) 

550 metrics = makeExampleMetrics() 

551 

552 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

553 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

554 conform=False) 

555 

556 # Configuration should have disabled checksum calculation 

557 datastore.put(metrics, ref) 

558 info = datastore.getStoredItemInfo(ref) 

559 self.assertIsNone(info.checksum) 

560 

561 # Remove put back but with checksums enabled explicitly 

562 datastore.remove(ref) 

563 datastore.useChecksum = True 

564 datastore.put(metrics, ref) 

565 

566 info = datastore.getStoredItemInfo(ref) 

567 self.assertIsNotNone(info.checksum) 

568 

569 

570class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase): 

571 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

572 

573 def setUp(self): 

574 # Override the working directory before calling the base class 

575 self.root = tempfile.mkdtemp(dir=TESTDIR) 

576 super().setUp() 

577 

578 def testCleanup(self): 

579 """Test that a failed formatter write does cleanup a partial file.""" 

580 metrics = makeExampleMetrics() 

581 datastore = self.makeDatastore() 

582 

583 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

584 

585 dimensions = self.universe.extract(("visit", "physical_filter")) 

586 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

587 

588 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

589 

590 # Determine where the file will end up (we assume Formatters use 

591 # the same file extension) 

592 expectedUri = datastore.getURI(ref, predict=True) 

593 self.assertEqual(expectedUri.fragment, "predicted") 

594 

595 expectedFile = expectedUri.path 

596 self.assertTrue(expectedFile.endswith(".yaml"), 

597 f"Is there a file extension in {expectedUri}") 

598 

599 # Try formatter that fails and formatter that fails and leaves 

600 # a file behind 

601 for formatter in (BadWriteFormatter, BadNoWriteFormatter): 

602 with self.subTest(formatter=formatter): 

603 

604 # Monkey patch the formatter 

605 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter, 

606 overwrite=True) 

607 

608 # Try to put the dataset, it should fail 

609 with self.assertRaises(Exception): 

610 datastore.put(metrics, ref) 

611 

612 # Check that there is no file on disk 

613 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

614 

615 # Check that there is a directory 

616 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)), 

617 f"Check for existence of directory {os.path.dirname(expectedFile)}") 

618 

619 # Force YamlFormatter and check that this time a file is written 

620 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter, 

621 overwrite=True) 

622 datastore.put(metrics, ref) 

623 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

624 datastore.remove(ref) 

625 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}") 

626 

627 

628class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase): 

629 """PosixDatastore specialization""" 

630 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml") 

631 uriScheme = "mem" 

632 hasUnsupportedPut = False 

633 ingestTransferModes = () 

634 isEphemeral = True 

635 rootKeys = None 

636 validationCanFail = False 

637 

638 

639class ChainedDatastoreTestCase(PosixDatastoreTestCase): 

640 """ChainedDatastore specialization using a POSIXDatastore""" 

641 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml") 

642 hasUnsupportedPut = False 

643 canIngestNoTransferAuto = False 

644 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto") 

645 isEphemeral = False 

646 rootKeys = (".datastores.1.root", ".datastores.2.root") 

647 validationCanFail = True 

648 

649 

650class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase): 

651 """ChainedDatastore specialization using all InMemoryDatastore""" 

652 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml") 

653 validationCanFail = False 

654 

655 

656class DatastoreConstraintsTests(DatastoreTestsBase): 

657 """Basic tests of constraints model of Datastores.""" 

658 

659 def testConstraints(self): 

660 """Test constraints model. Assumes that each test class has the 

661 same constraints.""" 

662 metrics = makeExampleMetrics() 

663 datastore = self.makeDatastore() 

664 

665 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

666 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

667 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

668 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

669 

670 # Write empty file suitable for ingest check 

671 testfile = tempfile.NamedTemporaryFile() 

672 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False), 

673 ("metric33", sc1, True), ("metric2", sc2, True)): 

674 with self.subTest(datasetTypeName=datasetTypeName): 

675 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False) 

676 if accepted: 

677 datastore.put(metrics, ref) 

678 self.assertTrue(datastore.exists(ref)) 

679 datastore.remove(ref) 

680 

681 # Try ingest 

682 if self.canIngest: 

683 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

684 self.assertTrue(datastore.exists(ref)) 

685 datastore.remove(ref) 

686 else: 

687 with self.assertRaises(DatasetTypeNotSupportedError): 

688 datastore.put(metrics, ref) 

689 self.assertFalse(datastore.exists(ref)) 

690 

691 # Again with ingest 

692 if self.canIngest: 

693 with self.assertRaises(DatasetTypeNotSupportedError): 

694 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

695 self.assertFalse(datastore.exists(ref)) 

696 

697 

698class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

699 """PosixDatastore specialization""" 

700 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml") 

701 canIngest = True 

702 

703 def setUp(self): 

704 # Override the working directory before calling the base class 

705 self.root = tempfile.mkdtemp(dir=TESTDIR) 

706 super().setUp() 

707 

708 

709class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

710 """InMemoryDatastore specialization""" 

711 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml") 

712 canIngest = False 

713 

714 

715class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase): 

716 """ChainedDatastore specialization using a POSIXDatastore and constraints 

717 at the ChainedDatstore """ 

718 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml") 

719 

720 

721class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase): 

722 """ChainedDatastore specialization using a POSIXDatastore""" 

723 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml") 

724 

725 

726class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase): 

727 """ChainedDatastore specialization using all InMemoryDatastore""" 

728 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml") 

729 canIngest = False 

730 

731 

732class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase): 

733 """Test that a chained datastore can control constraints per-datastore 

734 even if child datastore would accept.""" 

735 

736 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml") 

737 

738 def setUp(self): 

739 # Override the working directory before calling the base class 

740 self.root = tempfile.mkdtemp(dir=TESTDIR) 

741 super().setUp() 

742 

743 def testConstraints(self): 

744 """Test chained datastore constraints model.""" 

745 metrics = makeExampleMetrics() 

746 datastore = self.makeDatastore() 

747 

748 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

749 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

750 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

751 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

752 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"} 

753 

754 # Write empty file suitable for ingest check 

755 testfile = tempfile.NamedTemporaryFile() 

756 

757 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True), 

758 ("metric2", dataId1, sc1, (False, False, False), False), 

759 ("metric2", dataId2, sc1, (True, False, False), False), 

760 ("metric33", dataId2, sc2, (True, True, False), True), 

761 ("metric2", dataId1, sc2, (False, True, False), True)): 

762 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name): 

763 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId, 

764 conform=False) 

765 if any(accept): 

766 datastore.put(metrics, ref) 

767 self.assertTrue(datastore.exists(ref)) 

768 

769 # Check each datastore inside the chained datastore 

770 for childDatastore, expected in zip(datastore.datastores, accept): 

771 self.assertEqual(childDatastore.exists(ref), expected, 

772 f"Testing presence of {ref} in datastore {childDatastore.name}") 

773 

774 datastore.remove(ref) 

775 

776 # Check that ingest works 

777 if ingest: 

778 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

779 self.assertTrue(datastore.exists(ref)) 

780 

781 # Check each datastore inside the chained datastore 

782 for childDatastore, expected in zip(datastore.datastores, accept): 

783 # Ephemeral datastores means InMemory at the moment 

784 # and that does not accept ingest of files. 

785 if childDatastore.isEphemeral: 

786 expected = False 

787 self.assertEqual(childDatastore.exists(ref), expected, 

788 f"Testing presence of ingested {ref} in datastore" 

789 f" {childDatastore.name}") 

790 

791 datastore.remove(ref) 

792 else: 

793 with self.assertRaises(DatasetTypeNotSupportedError): 

794 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

795 

796 else: 

797 with self.assertRaises(DatasetTypeNotSupportedError): 

798 datastore.put(metrics, ref) 

799 self.assertFalse(datastore.exists(ref)) 

800 

801 # Again with ingest 

802 with self.assertRaises(DatasetTypeNotSupportedError): 

803 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

804 self.assertFalse(datastore.exists(ref)) 

805 

806 

807if __name__ == "__main__": 807 ↛ 808line 807 didn't jump to line 808, because the condition on line 807 was never true

808 unittest.main()