Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import shutil 

25import yaml 

26import tempfile 

27import lsst.utils.tests 

28 

29from lsst.utils import doImport 

30 

31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset 

32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError 

33from lsst.daf.butler.formatters.yaml import YamlFormatter 

34 

35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter, 

36 BadNoWriteFormatter, MetricsExample, DummyRegistry) 

37 

38 

39TESTDIR = os.path.dirname(__file__) 

40 

41 

42def makeExampleMetrics(use_none=False): 

43 if use_none: 

44 array = None 

45 else: 

46 array = [563, 234, 456.7] 

47 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

48 {"a": [1, 2, 3], 

49 "b": {"blue": 5, "red": "green"}}, 

50 array, 

51 ) 

52 

53 

54class TransactionTestError(Exception): 

55 """Specific error for transactions, to prevent misdiagnosing 

56 that might otherwise occur when a standard exception is used. 

57 """ 

58 pass 

59 

60 

61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper): 

62 """Support routines for datastore testing""" 

63 root = None 

64 

65 @classmethod 

66 def setUpClass(cls): 

67 # Storage Classes are fixed for all datastores in these tests 

68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") 

69 cls.storageClassFactory = StorageClassFactory() 

70 cls.storageClassFactory.addFromConfig(scConfigFile) 

71 

72 # Read the Datastore config so we can get the class 

73 # information (since we should not assume the constructor 

74 # name here, but rely on the configuration file itself) 

75 datastoreConfig = DatastoreConfig(cls.configFile) 

76 cls.datastoreType = doImport(datastoreConfig["cls"]) 

77 cls.universe = DimensionUniverse() 

78 

79 def setUp(self): 

80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig) 

81 

82 def tearDown(self): 

83 if self.root is not None and os.path.exists(self.root): 

84 shutil.rmtree(self.root, ignore_errors=True) 

85 

86 

87class DatastoreTests(DatastoreTestsBase): 

88 """Some basic tests of a simple datastore.""" 

89 

90 hasUnsupportedPut = True 

91 

92 def testConfigRoot(self): 

93 full = DatastoreConfig(self.configFile) 

94 config = DatastoreConfig(self.configFile, mergeDefaults=False) 

95 newroot = "/random/location" 

96 self.datastoreType.setConfigRoot(newroot, config, full) 

97 if self.rootKeys: 

98 for k in self.rootKeys: 

99 self.assertIn(newroot, config[k]) 

100 

101 def testConstructor(self): 

102 datastore = self.makeDatastore() 

103 self.assertIsNotNone(datastore) 

104 self.assertIs(datastore.isEphemeral, self.isEphemeral) 

105 

106 def testConfigurationValidation(self): 

107 datastore = self.makeDatastore() 

108 sc = self.storageClassFactory.getStorageClass("ThingOne") 

109 datastore.validateConfiguration([sc]) 

110 

111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo") 

112 if self.validationCanFail: 

113 with self.assertRaises(DatastoreValidationError): 

114 datastore.validateConfiguration([sc2], logFailures=True) 

115 

116 dimensions = self.universe.extract(("visit", "physical_filter")) 

117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

119 datastore.validateConfiguration([ref]) 

120 

121 def testParameterValidation(self): 

122 """Check that parameters are validated""" 

123 sc = self.storageClassFactory.getStorageClass("ThingOne") 

124 dimensions = self.universe.extract(("visit", "physical_filter")) 

125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

127 datastore = self.makeDatastore() 

128 data = {1: 2, 3: 4} 

129 datastore.put(data, ref) 

130 newdata = datastore.get(ref) 

131 self.assertEqual(data, newdata) 

132 with self.assertRaises(KeyError): 

133 newdata = datastore.get(ref, parameters={"missing": 5}) 

134 

135 def testBasicPutGet(self): 

136 metrics = makeExampleMetrics() 

137 datastore = self.makeDatastore() 

138 

139 # Create multiple storage classes for testing different formulations 

140 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

141 for sc in ("StructuredData", 

142 "StructuredDataJson", 

143 "StructuredDataPickle")] 

144 

145 dimensions = self.universe.extract(("visit", "physical_filter")) 

146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

147 

148 for sc in storageClasses: 

149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

150 print("Using storageClass: {}".format(sc.name)) 

151 datastore.put(metrics, ref) 

152 

153 # Does it exist? 

154 self.assertTrue(datastore.exists(ref)) 

155 

156 # Get 

157 metricsOut = datastore.get(ref, parameters=None) 

158 self.assertEqual(metrics, metricsOut) 

159 

160 uri = datastore.getURI(ref) 

161 self.assertEqual(uri.scheme, self.uriScheme) 

162 

163 # Get a component -- we need to construct new refs for them 

164 # with derived storage classes but with parent ID 

165 for comp in ("data", "output"): 

166 compRef = ref.makeComponentRef(comp) 

167 output = datastore.get(compRef) 

168 self.assertEqual(output, getattr(metricsOut, comp)) 

169 

170 uri = datastore.getURI(compRef) 

171 self.assertEqual(uri.scheme, self.uriScheme) 

172 

173 storageClass = sc 

174 

175 # Check that we can put a metric with None in a component and 

176 # get it back as None 

177 metricsNone = makeExampleMetrics(use_none=True) 

178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"} 

179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False) 

180 datastore.put(metricsNone, refNone) 

181 

182 comp = "data" 

183 for comp in ("data", "output"): 

184 compRef = refNone.makeComponentRef(comp) 

185 output = datastore.get(compRef) 

186 self.assertEqual(output, getattr(metricsNone, comp)) 

187 

188 # Check that a put fails if the dataset type is not supported 

189 if self.hasUnsupportedPut: 

190 sc = StorageClass("UnsupportedSC", pytype=type(metrics)) 

191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId) 

192 with self.assertRaises(DatasetTypeNotSupportedError): 

193 datastore.put(metrics, ref) 

194 

195 # These should raise 

196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000) 

197 with self.assertRaises(FileNotFoundError): 

198 # non-existing file 

199 datastore.get(ref) 

200 

201 # Get a URI from it 

202 uri = datastore.getURI(ref, predict=True) 

203 self.assertEqual(uri.scheme, self.uriScheme) 

204 

205 with self.assertRaises(FileNotFoundError): 

206 datastore.getURI(ref) 

207 

208 def testCompositePutGet(self): 

209 metrics = makeExampleMetrics() 

210 datastore = self.makeDatastore() 

211 

212 # Create multiple storage classes for testing different formulations 

213 # of composites 

214 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

215 for sc in ("StructuredComposite", 

216 "StructuredCompositeTestA", 

217 "StructuredCompositeTestB")] 

218 

219 dimensions = self.universe.extract(("visit", "physical_filter")) 

220 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

221 

222 for sc in storageClasses: 

223 print("Using storageClass: {}".format(sc.name)) 

224 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, 

225 conform=False) 

226 

227 components = sc.assembler().disassemble(metrics) 

228 self.assertTrue(components) 

229 

230 compsRead = {} 

231 for compName, compInfo in components.items(): 

232 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

233 components[compName].storageClass, dataId, 

234 conform=False) 

235 

236 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name)) 

237 datastore.put(compInfo.component, compRef) 

238 

239 uri = datastore.getURI(compRef) 

240 self.assertEqual(uri.scheme, self.uriScheme) 

241 

242 compsRead[compName] = datastore.get(compRef) 

243 

244 # We can generate identical files for each storage class 

245 # so remove the component here 

246 datastore.remove(compRef) 

247 

248 # combine all the components we read back into a new composite 

249 metricsOut = sc.assembler().assemble(compsRead) 

250 self.assertEqual(metrics, metricsOut) 

251 

252 def testRemove(self): 

253 metrics = makeExampleMetrics() 

254 datastore = self.makeDatastore() 

255 # Put 

256 dimensions = self.universe.extract(("visit", "physical_filter")) 

257 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"} 

258 

259 sc = self.storageClassFactory.getStorageClass("StructuredData") 

260 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

261 datastore.put(metrics, ref) 

262 

263 # Does it exist? 

264 self.assertTrue(datastore.exists(ref)) 

265 

266 # Get 

267 metricsOut = datastore.get(ref) 

268 self.assertEqual(metrics, metricsOut) 

269 # Remove 

270 datastore.remove(ref) 

271 

272 # Does it exist? 

273 self.assertFalse(datastore.exists(ref)) 

274 

275 # Do we now get a predicted URI? 

276 uri = datastore.getURI(ref, predict=True) 

277 self.assertEqual(uri.fragment, "predicted") 

278 

279 # Get should now fail 

280 with self.assertRaises(FileNotFoundError): 

281 datastore.get(ref) 

282 # Can only delete once 

283 with self.assertRaises(FileNotFoundError): 

284 datastore.remove(ref) 

285 

286 def testTransfer(self): 

287 metrics = makeExampleMetrics() 

288 

289 dimensions = self.universe.extract(("visit", "physical_filter")) 

290 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"} 

291 

292 sc = self.storageClassFactory.getStorageClass("StructuredData") 

293 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

294 

295 inputDatastore = self.makeDatastore("test_input_datastore") 

296 outputDatastore = self.makeDatastore("test_output_datastore") 

297 

298 inputDatastore.put(metrics, ref) 

299 outputDatastore.transfer(inputDatastore, ref) 

300 

301 metricsOut = outputDatastore.get(ref) 

302 self.assertEqual(metrics, metricsOut) 

303 

304 def testBasicTransaction(self): 

305 datastore = self.makeDatastore() 

306 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

307 dimensions = self.universe.extract(("visit", "physical_filter")) 

308 nDatasets = 6 

309 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)] 

310 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False), 

311 makeExampleMetrics(),) 

312 for dataId in dataIds] 

313 succeed = data[:nDatasets//2] 

314 fail = data[nDatasets//2:] 

315 # All datasets added in this transaction should continue to exist 

316 with datastore.transaction(): 

317 for ref, metrics in succeed: 

318 datastore.put(metrics, ref) 

319 # Whereas datasets added in this transaction should not 

320 with self.assertRaises(TransactionTestError): 

321 with datastore.transaction(): 

322 for ref, metrics in fail: 

323 datastore.put(metrics, ref) 

324 raise TransactionTestError("This should propagate out of the context manager") 

325 # Check for datasets that should exist 

326 for ref, metrics in succeed: 

327 # Does it exist? 

328 self.assertTrue(datastore.exists(ref)) 

329 # Get 

330 metricsOut = datastore.get(ref, parameters=None) 

331 self.assertEqual(metrics, metricsOut) 

332 # URI 

333 uri = datastore.getURI(ref) 

334 self.assertEqual(uri.scheme, self.uriScheme) 

335 # Check for datasets that should not exist 

336 for ref, _ in fail: 

337 # These should raise 

338 with self.assertRaises(FileNotFoundError): 

339 # non-existing file 

340 datastore.get(ref) 

341 with self.assertRaises(FileNotFoundError): 

342 datastore.getURI(ref) 

343 

344 def testNestedTransaction(self): 

345 datastore = self.makeDatastore() 

346 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

347 dimensions = self.universe.extract(("visit", "physical_filter")) 

348 metrics = makeExampleMetrics() 

349 

350 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

351 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

352 conform=False) 

353 datastore.put(metrics, refBefore) 

354 with self.assertRaises(TransactionTestError): 

355 with datastore.transaction(): 

356 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"} 

357 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

358 conform=False) 

359 datastore.put(metrics, refOuter) 

360 with datastore.transaction(): 

361 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"} 

362 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

363 conform=False) 

364 datastore.put(metrics, refInner) 

365 # All datasets should exist 

366 for ref in (refBefore, refOuter, refInner): 

367 metricsOut = datastore.get(ref, parameters=None) 

368 self.assertEqual(metrics, metricsOut) 

369 raise TransactionTestError("This should roll back the transaction") 

370 # Dataset(s) inserted before the transaction should still exist 

371 metricsOut = datastore.get(refBefore, parameters=None) 

372 self.assertEqual(metrics, metricsOut) 

373 # But all datasets inserted during the (rolled back) transaction 

374 # should be gone 

375 with self.assertRaises(FileNotFoundError): 

376 datastore.get(refOuter) 

377 with self.assertRaises(FileNotFoundError): 

378 datastore.get(refInner) 

379 

380 def _prepareIngestTest(self): 

381 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

382 dimensions = self.universe.extract(("visit", "physical_filter")) 

383 metrics = makeExampleMetrics() 

384 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

385 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

386 return metrics, ref 

387 

388 def runIngestTest(self, func, expectOutput=True): 

389 metrics, ref = self._prepareIngestTest() 

390 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path: 

391 with open(path, 'w') as fd: 

392 yaml.dump(metrics._asdict(), stream=fd) 

393 func(metrics, path, ref) 

394 

395 def testIngestNoTransfer(self): 

396 """Test ingesting existing files with no transfer. 

397 """ 

398 for mode in (None, "auto"): 

399 

400 # Some datastores have auto but can't do in place transfer 

401 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto: 

402 continue 

403 

404 with self.subTest(mode=mode): 

405 datastore = self.makeDatastore() 

406 

407 def succeed(obj, path, ref): 

408 """Ingest a file already in the datastore root.""" 

409 # first move it into the root, and adjust the path 

410 # accordingly 

411 path = shutil.copy(path, datastore.root) 

412 path = os.path.relpath(path, start=datastore.root) 

413 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

414 self.assertEqual(obj, datastore.get(ref)) 

415 

416 def failInputDoesNotExist(obj, path, ref): 

417 """Can't ingest files if we're given a bad path.""" 

418 with self.assertRaises(FileNotFoundError): 

419 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref), 

420 transfer=mode) 

421 self.assertFalse(datastore.exists(ref)) 

422 

423 def failOutsideRoot(obj, path, ref): 

424 """Can't ingest files outside of datastore root unless 

425 auto.""" 

426 if mode == "auto": 

427 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

428 self.assertTrue(datastore.exists(ref)) 

429 else: 

430 with self.assertRaises(RuntimeError): 

431 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

432 self.assertFalse(datastore.exists(ref)) 

433 

434 def failNotImplemented(obj, path, ref): 

435 with self.assertRaises(NotImplementedError): 

436 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

437 

438 if mode in self.ingestTransferModes: 

439 self.runIngestTest(failOutsideRoot) 

440 self.runIngestTest(failInputDoesNotExist) 

441 self.runIngestTest(succeed) 

442 else: 

443 self.runIngestTest(failNotImplemented) 

444 

445 def testIngestTransfer(self): 

446 """Test ingesting existing files after transferring them. 

447 """ 

448 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"): 

449 with self.subTest(mode=mode): 

450 datastore = self.makeDatastore(mode) 

451 

452 def succeed(obj, path, ref): 

453 """Ingest a file by transferring it to the template 

454 location.""" 

455 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

456 self.assertEqual(obj, datastore.get(ref)) 

457 

458 def failInputDoesNotExist(obj, path, ref): 

459 """Can't ingest files if we're given a bad path.""" 

460 with self.assertRaises(FileNotFoundError): 

461 # Ensure the file does not look like it is in 

462 # datastore for auto mode 

463 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref), 

464 transfer=mode) 

465 self.assertFalse(datastore.exists(ref)) 

466 

467 def failOutputExists(obj, path, ref): 

468 """Can't ingest files if transfer destination already 

469 exists.""" 

470 with self.assertRaises(FileExistsError): 

471 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

472 self.assertFalse(datastore.exists(ref)) 

473 

474 def failNotImplemented(obj, path, ref): 

475 with self.assertRaises(NotImplementedError): 

476 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

477 

478 if mode in self.ingestTransferModes: 

479 self.runIngestTest(failInputDoesNotExist) 

480 self.runIngestTest(succeed, expectOutput=(mode != "move")) 

481 self.runIngestTest(failOutputExists) 

482 else: 

483 self.runIngestTest(failNotImplemented) 

484 

485 def testIngestSymlinkOfSymlink(self): 

486 """Special test for symlink to a symlink ingest""" 

487 metrics, ref = self._prepareIngestTest() 

488 # The aim of this test is to create a dataset on disk, then 

489 # create a symlink to it and finally ingest the symlink such that 

490 # the symlink in the datastore points to the original dataset. 

491 for mode in ("symlink", "relsymlink"): 

492 if mode not in self.ingestTransferModes: 

493 continue 

494 

495 print(f"Trying mode {mode}") 

496 with lsst.utils.tests.getTempFilePath(".yaml") as realpath: 

497 with open(realpath, 'w') as fd: 

498 yaml.dump(metrics._asdict(), stream=fd) 

499 with lsst.utils.tests.getTempFilePath(".yaml") as sympath: 

500 os.symlink(os.path.abspath(realpath), sympath) 

501 

502 datastore = self.makeDatastore() 

503 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode) 

504 

505 uri = datastore.getURI(ref) 

506 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}") 

507 self.assertTrue(os.path.islink(uri.path)) 

508 

509 linkTarget = os.readlink(uri.path) 

510 if mode == "relsymlink": 

511 self.assertFalse(os.path.isabs(linkTarget)) 

512 else: 

513 self.assertEqual(linkTarget, os.path.abspath(realpath)) 

514 

515 # Check that we can get the dataset back regardless of mode 

516 metric2 = datastore.get(ref) 

517 self.assertEqual(metric2, metrics) 

518 

519 # Cleanup the file for next time round loop 

520 # since it will get the same file name in store 

521 datastore.remove(ref) 

522 

523 

524class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase): 

525 """PosixDatastore specialization""" 

526 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

527 uriScheme = "file" 

528 canIngestNoTransferAuto = True 

529 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto") 

530 isEphemeral = False 

531 rootKeys = ("root",) 

532 validationCanFail = True 

533 

534 def setUp(self): 

535 # Override the working directory before calling the base class 

536 self.root = tempfile.mkdtemp(dir=TESTDIR) 

537 super().setUp() 

538 

539 

540class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase): 

541 """Posix datastore tests but with checksums disabled.""" 

542 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml") 

543 

544 def testChecksum(self): 

545 """Ensure that checksums have not been calculated.""" 

546 

547 datastore = self.makeDatastore() 

548 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

549 dimensions = self.universe.extract(("visit", "physical_filter")) 

550 metrics = makeExampleMetrics() 

551 

552 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

553 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

554 conform=False) 

555 

556 # Configuration should have disabled checksum calculation 

557 datastore.put(metrics, ref) 

558 info = datastore.getStoredItemInfo(ref) 

559 self.assertIsNone(info.checksum) 

560 

561 # Remove put back but with checksums enabled explicitly 

562 datastore.remove(ref) 

563 datastore.useChecksum = True 

564 datastore.put(metrics, ref) 

565 

566 info = datastore.getStoredItemInfo(ref) 

567 self.assertIsNotNone(info.checksum) 

568 

569 

570class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase): 

571 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

572 

573 def setUp(self): 

574 # Override the working directory before calling the base class 

575 self.root = tempfile.mkdtemp(dir=TESTDIR) 

576 super().setUp() 

577 

578 def testCleanup(self): 

579 """Test that a failed formatter write does cleanup a partial file.""" 

580 metrics = makeExampleMetrics() 

581 datastore = self.makeDatastore() 

582 

583 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

584 

585 dimensions = self.universe.extract(("visit", "physical_filter")) 

586 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

587 

588 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

589 

590 # Determine where the file will end up (we assume Formatters use 

591 # the same file extension) 

592 expectedUri = datastore.getURI(ref, predict=True) 

593 self.assertEqual(expectedUri.fragment, "predicted") 

594 

595 expectedFile = expectedUri.path 

596 self.assertTrue(expectedFile.endswith(".yaml"), 

597 f"Is there a file extension in {expectedUri}") 

598 

599 # Try formatter that fails and formatter that fails and leaves 

600 # a file behind 

601 for formatter in (BadWriteFormatter, BadNoWriteFormatter): 

602 with self.subTest(formatter=formatter): 

603 

604 # Monkey patch the formatter 

605 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter, 

606 overwrite=True) 

607 

608 # Try to put the dataset, it should fail 

609 with self.assertRaises(Exception): 

610 datastore.put(metrics, ref) 

611 

612 # Check that there is no file on disk 

613 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

614 

615 # Check that there is a directory 

616 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)), 

617 f"Check for existence of directory {os.path.dirname(expectedFile)}") 

618 

619 # Force YamlFormatter and check that this time a file is written 

620 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter, 

621 overwrite=True) 

622 datastore.put(metrics, ref) 

623 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}") 

624 datastore.remove(ref) 

625 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}") 

626 

627 

628class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase): 

629 """PosixDatastore specialization""" 

630 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml") 

631 uriScheme = "mem" 

632 hasUnsupportedPut = False 

633 ingestTransferModes = () 

634 isEphemeral = True 

635 rootKeys = None 

636 validationCanFail = False 

637 

638 

639class ChainedDatastoreTestCase(PosixDatastoreTestCase): 

640 """ChainedDatastore specialization using a POSIXDatastore""" 

641 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml") 

642 hasUnsupportedPut = False 

643 canIngestNoTransferAuto = False 

644 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto") 

645 isEphemeral = False 

646 rootKeys = (".datastores.1.root", ".datastores.2.root") 

647 validationCanFail = True 

648 

649 

650class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase): 

651 """ChainedDatastore specialization using all InMemoryDatastore""" 

652 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml") 

653 validationCanFail = False 

654 

655 

656class DatastoreConstraintsTests(DatastoreTestsBase): 

657 """Basic tests of constraints model of Datastores.""" 

658 

659 def testConstraints(self): 

660 """Test constraints model. Assumes that each test class has the 

661 same constraints.""" 

662 metrics = makeExampleMetrics() 

663 datastore = self.makeDatastore() 

664 

665 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

666 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

667 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

668 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

669 

670 # Write empty file suitable for ingest check (JSON and YAML variants) 

671 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

672 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

673 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False), 

674 ("metric33", sc1, True), ("metric2", sc2, True)): 

675 # Choose different temp file depending on StorageClass 

676 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

677 

678 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name): 

679 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False) 

680 if accepted: 

681 datastore.put(metrics, ref) 

682 self.assertTrue(datastore.exists(ref)) 

683 datastore.remove(ref) 

684 

685 # Try ingest 

686 if self.canIngest: 

687 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

688 self.assertTrue(datastore.exists(ref)) 

689 datastore.remove(ref) 

690 else: 

691 with self.assertRaises(DatasetTypeNotSupportedError): 

692 datastore.put(metrics, ref) 

693 self.assertFalse(datastore.exists(ref)) 

694 

695 # Again with ingest 

696 if self.canIngest: 

697 with self.assertRaises(DatasetTypeNotSupportedError): 

698 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

699 self.assertFalse(datastore.exists(ref)) 

700 

701 

702class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

703 """PosixDatastore specialization""" 

704 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml") 

705 canIngest = True 

706 

707 def setUp(self): 

708 # Override the working directory before calling the base class 

709 self.root = tempfile.mkdtemp(dir=TESTDIR) 

710 super().setUp() 

711 

712 

713class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

714 """InMemoryDatastore specialization""" 

715 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml") 

716 canIngest = False 

717 

718 

719class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase): 

720 """ChainedDatastore specialization using a POSIXDatastore and constraints 

721 at the ChainedDatstore """ 

722 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml") 

723 

724 

725class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase): 

726 """ChainedDatastore specialization using a POSIXDatastore""" 

727 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml") 

728 

729 

730class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase): 

731 """ChainedDatastore specialization using all InMemoryDatastore""" 

732 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml") 

733 canIngest = False 

734 

735 

736class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase): 

737 """Test that a chained datastore can control constraints per-datastore 

738 even if child datastore would accept.""" 

739 

740 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml") 

741 

742 def setUp(self): 

743 # Override the working directory before calling the base class 

744 self.root = tempfile.mkdtemp(dir=TESTDIR) 

745 super().setUp() 

746 

747 def testConstraints(self): 

748 """Test chained datastore constraints model.""" 

749 metrics = makeExampleMetrics() 

750 datastore = self.makeDatastore() 

751 

752 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

753 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

754 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

755 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

756 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"} 

757 

758 # Write empty file suitable for ingest check (JSON and YAML variants) 

759 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

760 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

761 

762 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True), 

763 ("metric2", dataId1, sc1, (False, False, False), False), 

764 ("metric2", dataId2, sc1, (True, False, False), False), 

765 ("metric33", dataId2, sc2, (True, True, False), True), 

766 ("metric2", dataId1, sc2, (False, True, False), True)): 

767 

768 # Choose different temp file depending on StorageClass 

769 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

770 

771 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name): 

772 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId, 

773 conform=False) 

774 if any(accept): 

775 datastore.put(metrics, ref) 

776 self.assertTrue(datastore.exists(ref)) 

777 

778 # Check each datastore inside the chained datastore 

779 for childDatastore, expected in zip(datastore.datastores, accept): 

780 self.assertEqual(childDatastore.exists(ref), expected, 

781 f"Testing presence of {ref} in datastore {childDatastore.name}") 

782 

783 datastore.remove(ref) 

784 

785 # Check that ingest works 

786 if ingest: 

787 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

788 self.assertTrue(datastore.exists(ref)) 

789 

790 # Check each datastore inside the chained datastore 

791 for childDatastore, expected in zip(datastore.datastores, accept): 

792 # Ephemeral datastores means InMemory at the moment 

793 # and that does not accept ingest of files. 

794 if childDatastore.isEphemeral: 

795 expected = False 

796 self.assertEqual(childDatastore.exists(ref), expected, 

797 f"Testing presence of ingested {ref} in datastore" 

798 f" {childDatastore.name}") 

799 

800 datastore.remove(ref) 

801 else: 

802 with self.assertRaises(DatasetTypeNotSupportedError): 

803 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

804 

805 else: 

806 with self.assertRaises(DatasetTypeNotSupportedError): 

807 datastore.put(metrics, ref) 

808 self.assertFalse(datastore.exists(ref)) 

809 

810 # Again with ingest 

811 with self.assertRaises(DatasetTypeNotSupportedError): 

812 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

813 self.assertFalse(datastore.exists(ref)) 

814 

815 

816if __name__ == "__main__": 816 ↛ 817line 816 didn't jump to line 817, because the condition on line 816 was never true

817 unittest.main()