Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24import shutil 

25import yaml 

26import tempfile 

27import lsst.utils.tests 

28 

29from lsst.utils import doImport 

30 

31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset 

32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError 

33from lsst.daf.butler.formatters.yaml import YamlFormatter 

34 

35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter, 

36 BadNoWriteFormatter, MetricsExample, DummyRegistry) 

37 

38 

39TESTDIR = os.path.dirname(__file__) 

40 

41 

42def makeExampleMetrics(use_none=False): 

43 if use_none: 

44 array = None 

45 else: 

46 array = [563, 234, 456.7, 105, 2054, -1045] 

47 return MetricsExample({"AM1": 5.2, "AM2": 30.6}, 

48 {"a": [1, 2, 3], 

49 "b": {"blue": 5, "red": "green"}}, 

50 array, 

51 ) 

52 

53 

54class TransactionTestError(Exception): 

55 """Specific error for transactions, to prevent misdiagnosing 

56 that might otherwise occur when a standard exception is used. 

57 """ 

58 pass 

59 

60 

61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper): 

62 """Support routines for datastore testing""" 

63 root = None 

64 

65 @classmethod 

66 def setUpClass(cls): 

67 # Storage Classes are fixed for all datastores in these tests 

68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml") 

69 cls.storageClassFactory = StorageClassFactory() 

70 cls.storageClassFactory.addFromConfig(scConfigFile) 

71 

72 # Read the Datastore config so we can get the class 

73 # information (since we should not assume the constructor 

74 # name here, but rely on the configuration file itself) 

75 datastoreConfig = DatastoreConfig(cls.configFile) 

76 cls.datastoreType = doImport(datastoreConfig["cls"]) 

77 cls.universe = DimensionUniverse() 

78 

79 def setUp(self): 

80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig) 

81 

82 def tearDown(self): 

83 if self.root is not None and os.path.exists(self.root): 

84 shutil.rmtree(self.root, ignore_errors=True) 

85 

86 

87class DatastoreTests(DatastoreTestsBase): 

88 """Some basic tests of a simple datastore.""" 

89 

90 hasUnsupportedPut = True 

91 

92 def testConfigRoot(self): 

93 full = DatastoreConfig(self.configFile) 

94 config = DatastoreConfig(self.configFile, mergeDefaults=False) 

95 newroot = "/random/location" 

96 self.datastoreType.setConfigRoot(newroot, config, full) 

97 if self.rootKeys: 

98 for k in self.rootKeys: 

99 self.assertIn(newroot, config[k]) 

100 

101 def testConstructor(self): 

102 datastore = self.makeDatastore() 

103 self.assertIsNotNone(datastore) 

104 self.assertIs(datastore.isEphemeral, self.isEphemeral) 

105 

106 def testConfigurationValidation(self): 

107 datastore = self.makeDatastore() 

108 sc = self.storageClassFactory.getStorageClass("ThingOne") 

109 datastore.validateConfiguration([sc]) 

110 

111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo") 

112 if self.validationCanFail: 

113 with self.assertRaises(DatastoreValidationError): 

114 datastore.validateConfiguration([sc2], logFailures=True) 

115 

116 dimensions = self.universe.extract(("visit", "physical_filter")) 

117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

119 datastore.validateConfiguration([ref]) 

120 

121 def testParameterValidation(self): 

122 """Check that parameters are validated""" 

123 sc = self.storageClassFactory.getStorageClass("ThingOne") 

124 dimensions = self.universe.extract(("visit", "physical_filter")) 

125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

127 datastore = self.makeDatastore() 

128 data = {1: 2, 3: 4} 

129 datastore.put(data, ref) 

130 newdata = datastore.get(ref) 

131 self.assertEqual(data, newdata) 

132 with self.assertRaises(KeyError): 

133 newdata = datastore.get(ref, parameters={"missing": 5}) 

134 

135 def testBasicPutGet(self): 

136 metrics = makeExampleMetrics() 

137 datastore = self.makeDatastore() 

138 

139 # Create multiple storage classes for testing different formulations 

140 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

141 for sc in ("StructuredData", 

142 "StructuredDataJson", 

143 "StructuredDataPickle")] 

144 

145 dimensions = self.universe.extract(("visit", "physical_filter")) 

146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

147 

148 for sc in storageClasses: 

149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

150 print("Using storageClass: {}".format(sc.name)) 

151 datastore.put(metrics, ref) 

152 

153 # Does it exist? 

154 self.assertTrue(datastore.exists(ref)) 

155 

156 # Get 

157 metricsOut = datastore.get(ref, parameters=None) 

158 self.assertEqual(metrics, metricsOut) 

159 

160 uri = datastore.getURI(ref) 

161 self.assertEqual(uri.scheme, self.uriScheme) 

162 

163 # Get a component -- we need to construct new refs for them 

164 # with derived storage classes but with parent ID 

165 for comp in ("data", "output"): 

166 compRef = ref.makeComponentRef(comp) 

167 output = datastore.get(compRef) 

168 self.assertEqual(output, getattr(metricsOut, comp)) 

169 

170 uri = datastore.getURI(compRef) 

171 self.assertEqual(uri.scheme, self.uriScheme) 

172 

173 storageClass = sc 

174 

175 # Check that we can put a metric with None in a component and 

176 # get it back as None 

177 metricsNone = makeExampleMetrics(use_none=True) 

178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"} 

179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False) 

180 datastore.put(metricsNone, refNone) 

181 

182 comp = "data" 

183 for comp in ("data", "output"): 

184 compRef = refNone.makeComponentRef(comp) 

185 output = datastore.get(compRef) 

186 self.assertEqual(output, getattr(metricsNone, comp)) 

187 

188 # Check that a put fails if the dataset type is not supported 

189 if self.hasUnsupportedPut: 

190 sc = StorageClass("UnsupportedSC", pytype=type(metrics)) 

191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId) 

192 with self.assertRaises(DatasetTypeNotSupportedError): 

193 datastore.put(metrics, ref) 

194 

195 # These should raise 

196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000) 

197 with self.assertRaises(FileNotFoundError): 

198 # non-existing file 

199 datastore.get(ref) 

200 

201 # Get a URI from it 

202 uri = datastore.getURI(ref, predict=True) 

203 self.assertEqual(uri.scheme, self.uriScheme) 

204 

205 with self.assertRaises(FileNotFoundError): 

206 datastore.getURI(ref) 

207 

208 def testDisassembly(self): 

209 """Test disassembly within datastore.""" 

210 metrics = makeExampleMetrics() 

211 if self.isEphemeral: 

212 # in-memory datastore does not disassemble 

213 return 

214 

215 # Create multiple storage classes for testing different formulations 

216 # of composites. One of these will not disassemble to provide 

217 # a reference. 

218 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

219 for sc in ("StructuredComposite", 

220 "StructuredCompositeTestA", 

221 "StructuredCompositeTestB", 

222 "StructuredCompositeReadComp", 

223 "StructuredData", # No disassembly 

224 "StructuredCompositeReadCompNoDisassembly", 

225 )] 

226 

227 # Create the test datastore 

228 datastore = self.makeDatastore() 

229 

230 # Dummy dataId 

231 dimensions = self.universe.extract(("visit", "physical_filter")) 

232 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

233 

234 for i, sc in enumerate(storageClasses): 

235 with self.subTest(storageClass=sc.name): 

236 # Create a different dataset type each time round 

237 # so that a test failure in this subtest does not trigger 

238 # a cascade of tests because of file clashes 

239 ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId, 

240 conform=False) 

241 

242 disassembled = sc.name not in {"StructuredData", "StructuredCompositeReadCompNoDisassembly"} 

243 

244 datastore.put(metrics, ref) 

245 

246 baseURI, compURIs = datastore.getURIs(ref) 

247 if disassembled: 

248 self.assertIsNone(baseURI) 

249 self.assertEqual(set(compURIs), {"data", "output", "summary"}) 

250 else: 

251 self.assertIsNotNone(baseURI) 

252 self.assertEqual(compURIs, {}) 

253 

254 metrics_get = datastore.get(ref) 

255 self.assertEqual(metrics_get, metrics) 

256 

257 # Retrieve the composite with read parameter 

258 stop = 4 

259 metrics_get = datastore.get(ref, parameters={"slice": slice(stop)}) 

260 self.assertEqual(metrics_get.summary, metrics.summary) 

261 self.assertEqual(metrics_get.output, metrics.output) 

262 self.assertEqual(metrics_get.data, metrics.data[:stop]) 

263 

264 # Retrieve a component 

265 data = datastore.get(ref.makeComponentRef("data")) 

266 self.assertEqual(data, metrics.data) 

267 

268 # On supported storage classes attempt to access a read 

269 # only component 

270 if "ReadComp" in sc.name: 

271 cRef = ref.makeComponentRef("counter") 

272 counter = datastore.get(cRef) 

273 self.assertEqual(counter, len(metrics.data)) 

274 

275 counter = datastore.get(cRef, parameters={"slice": slice(stop)}) 

276 self.assertEqual(counter, stop) 

277 

278 datastore.remove(ref) 

279 

280 def testRegistryCompositePutGet(self): 

281 """Tests the case where registry disassembles and puts to datastore. 

282 """ 

283 metrics = makeExampleMetrics() 

284 datastore = self.makeDatastore() 

285 

286 # Create multiple storage classes for testing different formulations 

287 # of composites 

288 storageClasses = [self.storageClassFactory.getStorageClass(sc) 

289 for sc in ("StructuredComposite", 

290 "StructuredCompositeTestA", 

291 "StructuredCompositeTestB", 

292 )] 

293 

294 dimensions = self.universe.extract(("visit", "physical_filter")) 

295 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"} 

296 

297 for sc in storageClasses: 

298 print("Using storageClass: {}".format(sc.name)) 

299 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, 

300 conform=False) 

301 

302 components = sc.assembler().disassemble(metrics) 

303 self.assertTrue(components) 

304 

305 compsRead = {} 

306 for compName, compInfo in components.items(): 

307 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions, 

308 components[compName].storageClass, dataId, 

309 conform=False) 

310 

311 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name)) 

312 datastore.put(compInfo.component, compRef) 

313 

314 uri = datastore.getURI(compRef) 

315 self.assertEqual(uri.scheme, self.uriScheme) 

316 

317 compsRead[compName] = datastore.get(compRef) 

318 

319 # We can generate identical files for each storage class 

320 # so remove the component here 

321 datastore.remove(compRef) 

322 

323 # combine all the components we read back into a new composite 

324 metricsOut = sc.assembler().assemble(compsRead) 

325 self.assertEqual(metrics, metricsOut) 

326 

327 def testRemove(self): 

328 metrics = makeExampleMetrics() 

329 datastore = self.makeDatastore() 

330 # Put 

331 dimensions = self.universe.extract(("visit", "physical_filter")) 

332 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"} 

333 

334 sc = self.storageClassFactory.getStorageClass("StructuredData") 

335 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

336 datastore.put(metrics, ref) 

337 

338 # Does it exist? 

339 self.assertTrue(datastore.exists(ref)) 

340 

341 # Get 

342 metricsOut = datastore.get(ref) 

343 self.assertEqual(metrics, metricsOut) 

344 # Remove 

345 datastore.remove(ref) 

346 

347 # Does it exist? 

348 self.assertFalse(datastore.exists(ref)) 

349 

350 # Do we now get a predicted URI? 

351 uri = datastore.getURI(ref, predict=True) 

352 self.assertEqual(uri.fragment, "predicted") 

353 

354 # Get should now fail 

355 with self.assertRaises(FileNotFoundError): 

356 datastore.get(ref) 

357 # Can only delete once 

358 with self.assertRaises(FileNotFoundError): 

359 datastore.remove(ref) 

360 

361 def testTransfer(self): 

362 metrics = makeExampleMetrics() 

363 

364 dimensions = self.universe.extract(("visit", "physical_filter")) 

365 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"} 

366 

367 sc = self.storageClassFactory.getStorageClass("StructuredData") 

368 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False) 

369 

370 inputDatastore = self.makeDatastore("test_input_datastore") 

371 outputDatastore = self.makeDatastore("test_output_datastore") 

372 

373 inputDatastore.put(metrics, ref) 

374 outputDatastore.transfer(inputDatastore, ref) 

375 

376 metricsOut = outputDatastore.get(ref) 

377 self.assertEqual(metrics, metricsOut) 

378 

379 def testBasicTransaction(self): 

380 datastore = self.makeDatastore() 

381 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

382 dimensions = self.universe.extract(("visit", "physical_filter")) 

383 nDatasets = 6 

384 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)] 

385 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False), 

386 makeExampleMetrics(),) 

387 for dataId in dataIds] 

388 succeed = data[:nDatasets//2] 

389 fail = data[nDatasets//2:] 

390 # All datasets added in this transaction should continue to exist 

391 with datastore.transaction(): 

392 for ref, metrics in succeed: 

393 datastore.put(metrics, ref) 

394 # Whereas datasets added in this transaction should not 

395 with self.assertRaises(TransactionTestError): 

396 with datastore.transaction(): 

397 for ref, metrics in fail: 

398 datastore.put(metrics, ref) 

399 raise TransactionTestError("This should propagate out of the context manager") 

400 # Check for datasets that should exist 

401 for ref, metrics in succeed: 

402 # Does it exist? 

403 self.assertTrue(datastore.exists(ref)) 

404 # Get 

405 metricsOut = datastore.get(ref, parameters=None) 

406 self.assertEqual(metrics, metricsOut) 

407 # URI 

408 uri = datastore.getURI(ref) 

409 self.assertEqual(uri.scheme, self.uriScheme) 

410 # Check for datasets that should not exist 

411 for ref, _ in fail: 

412 # These should raise 

413 with self.assertRaises(FileNotFoundError): 

414 # non-existing file 

415 datastore.get(ref) 

416 with self.assertRaises(FileNotFoundError): 

417 datastore.getURI(ref) 

418 

419 def testNestedTransaction(self): 

420 datastore = self.makeDatastore() 

421 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

422 dimensions = self.universe.extract(("visit", "physical_filter")) 

423 metrics = makeExampleMetrics() 

424 

425 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

426 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

427 conform=False) 

428 datastore.put(metrics, refBefore) 

429 with self.assertRaises(TransactionTestError): 

430 with datastore.transaction(): 

431 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"} 

432 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

433 conform=False) 

434 datastore.put(metrics, refOuter) 

435 with datastore.transaction(): 

436 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"} 

437 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

438 conform=False) 

439 datastore.put(metrics, refInner) 

440 # All datasets should exist 

441 for ref in (refBefore, refOuter, refInner): 

442 metricsOut = datastore.get(ref, parameters=None) 

443 self.assertEqual(metrics, metricsOut) 

444 raise TransactionTestError("This should roll back the transaction") 

445 # Dataset(s) inserted before the transaction should still exist 

446 metricsOut = datastore.get(refBefore, parameters=None) 

447 self.assertEqual(metrics, metricsOut) 

448 # But all datasets inserted during the (rolled back) transaction 

449 # should be gone 

450 with self.assertRaises(FileNotFoundError): 

451 datastore.get(refOuter) 

452 with self.assertRaises(FileNotFoundError): 

453 datastore.get(refInner) 

454 

455 def _prepareIngestTest(self): 

456 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

457 dimensions = self.universe.extract(("visit", "physical_filter")) 

458 metrics = makeExampleMetrics() 

459 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

460 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

461 return metrics, ref 

462 

463 def runIngestTest(self, func, expectOutput=True): 

464 metrics, ref = self._prepareIngestTest() 

465 # The file will be deleted after the test. 

466 # For symlink tests this leads to a situation where the datastore 

467 # points to a file that does not exist. This will make os.path.exist 

468 # return False but then the new symlink will fail with 

469 # FileExistsError later in the code so the test still passes. 

470 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path: 

471 with open(path, 'w') as fd: 

472 yaml.dump(metrics._asdict(), stream=fd) 

473 func(metrics, path, ref) 

474 

475 def testIngestNoTransfer(self): 

476 """Test ingesting existing files with no transfer. 

477 """ 

478 for mode in (None, "auto"): 

479 

480 # Some datastores have auto but can't do in place transfer 

481 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto: 

482 continue 

483 

484 with self.subTest(mode=mode): 

485 datastore = self.makeDatastore() 

486 

487 def succeed(obj, path, ref): 

488 """Ingest a file already in the datastore root.""" 

489 # first move it into the root, and adjust the path 

490 # accordingly 

491 path = shutil.copy(path, datastore.root.ospath) 

492 path = os.path.relpath(path, start=datastore.root.ospath) 

493 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

494 self.assertEqual(obj, datastore.get(ref)) 

495 

496 def failInputDoesNotExist(obj, path, ref): 

497 """Can't ingest files if we're given a bad path.""" 

498 with self.assertRaises(FileNotFoundError): 

499 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref), 

500 transfer=mode) 

501 self.assertFalse(datastore.exists(ref)) 

502 

503 def failOutsideRoot(obj, path, ref): 

504 """Can't ingest files outside of datastore root unless 

505 auto.""" 

506 if mode == "auto": 

507 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

508 self.assertTrue(datastore.exists(ref)) 

509 else: 

510 with self.assertRaises(RuntimeError): 

511 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

512 self.assertFalse(datastore.exists(ref)) 

513 

514 def failNotImplemented(obj, path, ref): 

515 with self.assertRaises(NotImplementedError): 

516 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode) 

517 

518 if mode in self.ingestTransferModes: 

519 self.runIngestTest(failOutsideRoot) 

520 self.runIngestTest(failInputDoesNotExist) 

521 self.runIngestTest(succeed) 

522 else: 

523 self.runIngestTest(failNotImplemented) 

524 

525 def testIngestTransfer(self): 

526 """Test ingesting existing files after transferring them. 

527 """ 

528 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"): 

529 with self.subTest(mode=mode): 

530 datastore = self.makeDatastore(mode) 

531 

532 def succeed(obj, path, ref): 

533 """Ingest a file by transferring it to the template 

534 location.""" 

535 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

536 self.assertEqual(obj, datastore.get(ref)) 

537 

538 def failInputDoesNotExist(obj, path, ref): 

539 """Can't ingest files if we're given a bad path.""" 

540 with self.assertRaises(FileNotFoundError): 

541 # Ensure the file does not look like it is in 

542 # datastore for auto mode 

543 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref), 

544 transfer=mode) 

545 self.assertFalse(datastore.exists(ref), f"Checking not in datastore using mode {mode}") 

546 

547 def failOutputExists(obj, path, ref): 

548 """Can't ingest files if transfer destination already 

549 exists.""" 

550 with self.assertRaises(FileExistsError): 

551 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

552 self.assertFalse(datastore.exists(ref), f"Checking not in datastore using mode {mode}") 

553 

554 def failNotImplemented(obj, path, ref): 

555 with self.assertRaises(NotImplementedError): 

556 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode) 

557 

558 if mode in self.ingestTransferModes: 

559 self.runIngestTest(failInputDoesNotExist) 

560 self.runIngestTest(succeed, expectOutput=(mode != "move")) 

561 self.runIngestTest(failOutputExists) 

562 else: 

563 self.runIngestTest(failNotImplemented) 

564 

565 def testIngestSymlinkOfSymlink(self): 

566 """Special test for symlink to a symlink ingest""" 

567 metrics, ref = self._prepareIngestTest() 

568 # The aim of this test is to create a dataset on disk, then 

569 # create a symlink to it and finally ingest the symlink such that 

570 # the symlink in the datastore points to the original dataset. 

571 for mode in ("symlink", "relsymlink"): 

572 if mode not in self.ingestTransferModes: 

573 continue 

574 

575 print(f"Trying mode {mode}") 

576 with lsst.utils.tests.getTempFilePath(".yaml") as realpath: 

577 with open(realpath, 'w') as fd: 

578 yaml.dump(metrics._asdict(), stream=fd) 

579 with lsst.utils.tests.getTempFilePath(".yaml") as sympath: 

580 os.symlink(os.path.abspath(realpath), sympath) 

581 

582 datastore = self.makeDatastore() 

583 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode) 

584 

585 uri = datastore.getURI(ref) 

586 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}") 

587 self.assertTrue(os.path.islink(uri.ospath), f"Check {uri} is a symlink") 

588 

589 linkTarget = os.readlink(uri.ospath) 

590 if mode == "relsymlink": 

591 self.assertFalse(os.path.isabs(linkTarget)) 

592 else: 

593 self.assertEqual(linkTarget, os.path.abspath(realpath)) 

594 

595 # Check that we can get the dataset back regardless of mode 

596 metric2 = datastore.get(ref) 

597 self.assertEqual(metric2, metrics) 

598 

599 # Cleanup the file for next time round loop 

600 # since it will get the same file name in store 

601 datastore.remove(ref) 

602 

603 

604class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase): 

605 """PosixDatastore specialization""" 

606 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

607 uriScheme = "file" 

608 canIngestNoTransferAuto = True 

609 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto") 

610 isEphemeral = False 

611 rootKeys = ("root",) 

612 validationCanFail = True 

613 

614 def setUp(self): 

615 # Override the working directory before calling the base class 

616 self.root = tempfile.mkdtemp(dir=TESTDIR) 

617 super().setUp() 

618 

619 

620class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase): 

621 """Posix datastore tests but with checksums disabled.""" 

622 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml") 

623 

624 def testChecksum(self): 

625 """Ensure that checksums have not been calculated.""" 

626 

627 datastore = self.makeDatastore() 

628 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

629 dimensions = self.universe.extract(("visit", "physical_filter")) 

630 metrics = makeExampleMetrics() 

631 

632 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"} 

633 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, 

634 conform=False) 

635 

636 # Configuration should have disabled checksum calculation 

637 datastore.put(metrics, ref) 

638 infos = datastore.getStoredItemsInfo(ref) 

639 self.assertIsNone(infos[0].checksum) 

640 

641 # Remove put back but with checksums enabled explicitly 

642 datastore.remove(ref) 

643 datastore.useChecksum = True 

644 datastore.put(metrics, ref) 

645 

646 infos = datastore.getStoredItemsInfo(ref) 

647 self.assertIsNotNone(infos[0].checksum) 

648 

649 

650class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase): 

651 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml") 

652 

653 def setUp(self): 

654 # Override the working directory before calling the base class 

655 self.root = tempfile.mkdtemp(dir=TESTDIR) 

656 super().setUp() 

657 

658 def testCleanup(self): 

659 """Test that a failed formatter write does cleanup a partial file.""" 

660 metrics = makeExampleMetrics() 

661 datastore = self.makeDatastore() 

662 

663 storageClass = self.storageClassFactory.getStorageClass("StructuredData") 

664 

665 dimensions = self.universe.extract(("visit", "physical_filter")) 

666 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"} 

667 

668 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False) 

669 

670 # Determine where the file will end up (we assume Formatters use 

671 # the same file extension) 

672 expectedUri = datastore.getURI(ref, predict=True) 

673 self.assertEqual(expectedUri.fragment, "predicted") 

674 

675 self.assertEqual(expectedUri.getExtension(), ".yaml", 

676 f"Is there a file extension in {expectedUri}") 

677 

678 # Try formatter that fails and formatter that fails and leaves 

679 # a file behind 

680 for formatter in (BadWriteFormatter, BadNoWriteFormatter): 

681 with self.subTest(formatter=formatter): 

682 

683 # Monkey patch the formatter 

684 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter, 

685 overwrite=True) 

686 

687 # Try to put the dataset, it should fail 

688 with self.assertRaises(Exception): 

689 datastore.put(metrics, ref) 

690 

691 # Check that there is no file on disk 

692 self.assertFalse(expectedUri.exists(), f"Check for existence of {expectedUri}") 

693 

694 # Check that there is a directory 

695 dir = expectedUri.dirname() 

696 self.assertTrue(dir.exists(), 

697 f"Check for existence of directory {dir}") 

698 

699 # Force YamlFormatter and check that this time a file is written 

700 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter, 

701 overwrite=True) 

702 datastore.put(metrics, ref) 

703 self.assertTrue(expectedUri.exists(), f"Check for existence of {expectedUri}") 

704 datastore.remove(ref) 

705 self.assertFalse(expectedUri.exists(), f"Check for existence of now removed {expectedUri}") 

706 

707 

708class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase): 

709 """PosixDatastore specialization""" 

710 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml") 

711 uriScheme = "mem" 

712 hasUnsupportedPut = False 

713 ingestTransferModes = () 

714 isEphemeral = True 

715 rootKeys = None 

716 validationCanFail = False 

717 

718 

719class ChainedDatastoreTestCase(PosixDatastoreTestCase): 

720 """ChainedDatastore specialization using a POSIXDatastore""" 

721 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml") 

722 hasUnsupportedPut = False 

723 canIngestNoTransferAuto = False 

724 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto") 

725 isEphemeral = False 

726 rootKeys = (".datastores.1.root", ".datastores.2.root") 

727 validationCanFail = True 

728 

729 

730class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase): 

731 """ChainedDatastore specialization using all InMemoryDatastore""" 

732 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml") 

733 validationCanFail = False 

734 

735 

736class DatastoreConstraintsTests(DatastoreTestsBase): 

737 """Basic tests of constraints model of Datastores.""" 

738 

739 def testConstraints(self): 

740 """Test constraints model. Assumes that each test class has the 

741 same constraints.""" 

742 metrics = makeExampleMetrics() 

743 datastore = self.makeDatastore() 

744 

745 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

746 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

747 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

748 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

749 

750 # Write empty file suitable for ingest check (JSON and YAML variants) 

751 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

752 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

753 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False), 

754 ("metric33", sc1, True), ("metric2", sc2, True)): 

755 # Choose different temp file depending on StorageClass 

756 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

757 

758 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name): 

759 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False) 

760 if accepted: 

761 datastore.put(metrics, ref) 

762 self.assertTrue(datastore.exists(ref)) 

763 datastore.remove(ref) 

764 

765 # Try ingest 

766 if self.canIngest: 

767 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

768 self.assertTrue(datastore.exists(ref)) 

769 datastore.remove(ref) 

770 else: 

771 with self.assertRaises(DatasetTypeNotSupportedError): 

772 datastore.put(metrics, ref) 

773 self.assertFalse(datastore.exists(ref)) 

774 

775 # Again with ingest 

776 if self.canIngest: 

777 with self.assertRaises(DatasetTypeNotSupportedError): 

778 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

779 self.assertFalse(datastore.exists(ref)) 

780 

781 

782class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

783 """PosixDatastore specialization""" 

784 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml") 

785 canIngest = True 

786 

787 def setUp(self): 

788 # Override the working directory before calling the base class 

789 self.root = tempfile.mkdtemp(dir=TESTDIR) 

790 super().setUp() 

791 

792 

793class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase): 

794 """InMemoryDatastore specialization""" 

795 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml") 

796 canIngest = False 

797 

798 

799class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase): 

800 """ChainedDatastore specialization using a POSIXDatastore and constraints 

801 at the ChainedDatstore """ 

802 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml") 

803 

804 

805class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase): 

806 """ChainedDatastore specialization using a POSIXDatastore""" 

807 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml") 

808 

809 

810class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase): 

811 """ChainedDatastore specialization using all InMemoryDatastore""" 

812 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml") 

813 canIngest = False 

814 

815 

816class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase): 

817 """Test that a chained datastore can control constraints per-datastore 

818 even if child datastore would accept.""" 

819 

820 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml") 

821 

822 def setUp(self): 

823 # Override the working directory before calling the base class 

824 self.root = tempfile.mkdtemp(dir=TESTDIR) 

825 super().setUp() 

826 

827 def testConstraints(self): 

828 """Test chained datastore constraints model.""" 

829 metrics = makeExampleMetrics() 

830 datastore = self.makeDatastore() 

831 

832 sc1 = self.storageClassFactory.getStorageClass("StructuredData") 

833 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson") 

834 dimensions = self.universe.extract(("visit", "physical_filter", "instrument")) 

835 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"} 

836 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"} 

837 

838 # Write empty file suitable for ingest check (JSON and YAML variants) 

839 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml") 

840 testfile_j = tempfile.NamedTemporaryFile(suffix=".json") 

841 

842 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True), 

843 ("metric2", dataId1, sc1, (False, False, False), False), 

844 ("metric2", dataId2, sc1, (True, False, False), False), 

845 ("metric33", dataId2, sc2, (True, True, False), True), 

846 ("metric2", dataId1, sc2, (False, True, False), True)): 

847 

848 # Choose different temp file depending on StorageClass 

849 testfile = testfile_j if sc.name.endswith("Json") else testfile_y 

850 

851 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name): 

852 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId, 

853 conform=False) 

854 if any(accept): 

855 datastore.put(metrics, ref) 

856 self.assertTrue(datastore.exists(ref)) 

857 

858 # Check each datastore inside the chained datastore 

859 for childDatastore, expected in zip(datastore.datastores, accept): 

860 self.assertEqual(childDatastore.exists(ref), expected, 

861 f"Testing presence of {ref} in datastore {childDatastore.name}") 

862 

863 datastore.remove(ref) 

864 

865 # Check that ingest works 

866 if ingest: 

867 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

868 self.assertTrue(datastore.exists(ref)) 

869 

870 # Check each datastore inside the chained datastore 

871 for childDatastore, expected in zip(datastore.datastores, accept): 

872 # Ephemeral datastores means InMemory at the moment 

873 # and that does not accept ingest of files. 

874 if childDatastore.isEphemeral: 

875 expected = False 

876 self.assertEqual(childDatastore.exists(ref), expected, 

877 f"Testing presence of ingested {ref} in datastore" 

878 f" {childDatastore.name}") 

879 

880 datastore.remove(ref) 

881 else: 

882 with self.assertRaises(DatasetTypeNotSupportedError): 

883 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

884 

885 else: 

886 with self.assertRaises(DatasetTypeNotSupportedError): 

887 datastore.put(metrics, ref) 

888 self.assertFalse(datastore.exists(ref)) 

889 

890 # Again with ingest 

891 with self.assertRaises(DatasetTypeNotSupportedError): 

892 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link") 

893 self.assertFalse(datastore.exists(ref)) 

894 

895 

896if __name__ == "__main__": 896 ↛ 897line 896 didn't jump to line 897, because the condition on line 896 was never true

897 unittest.main()