Coverage for tests/test_datastore.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import unittest
24import shutil
25import yaml
26import tempfile
27import lsst.utils.tests
29from lsst.utils import doImport
31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset
32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError
33from lsst.daf.butler.formatters.yaml import YamlFormatter
35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter,
36 BadNoWriteFormatter, MetricsExample, DummyRegistry)
39TESTDIR = os.path.dirname(__file__)
42def makeExampleMetrics(use_none=False):
43 if use_none:
44 array = None
45 else:
46 array = [563, 234, 456.7]
47 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
48 {"a": [1, 2, 3],
49 "b": {"blue": 5, "red": "green"}},
50 array,
51 )
54class TransactionTestError(Exception):
55 """Specific error for transactions, to prevent misdiagnosing
56 that might otherwise occur when a standard exception is used.
57 """
58 pass
61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper):
62 """Support routines for datastore testing"""
63 root = None
65 @classmethod
66 def setUpClass(cls):
67 # Storage Classes are fixed for all datastores in these tests
68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
69 cls.storageClassFactory = StorageClassFactory()
70 cls.storageClassFactory.addFromConfig(scConfigFile)
72 # Read the Datastore config so we can get the class
73 # information (since we should not assume the constructor
74 # name here, but rely on the configuration file itself)
75 datastoreConfig = DatastoreConfig(cls.configFile)
76 cls.datastoreType = doImport(datastoreConfig["cls"])
77 cls.universe = DimensionUniverse()
79 def setUp(self):
80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig)
82 def tearDown(self):
83 if self.root is not None and os.path.exists(self.root):
84 shutil.rmtree(self.root, ignore_errors=True)
87class DatastoreTests(DatastoreTestsBase):
88 """Some basic tests of a simple datastore."""
90 hasUnsupportedPut = True
92 def testConfigRoot(self):
93 full = DatastoreConfig(self.configFile)
94 config = DatastoreConfig(self.configFile, mergeDefaults=False)
95 newroot = "/random/location"
96 self.datastoreType.setConfigRoot(newroot, config, full)
97 if self.rootKeys:
98 for k in self.rootKeys:
99 self.assertIn(newroot, config[k])
101 def testConstructor(self):
102 datastore = self.makeDatastore()
103 self.assertIsNotNone(datastore)
104 self.assertIs(datastore.isEphemeral, self.isEphemeral)
106 def testConfigurationValidation(self):
107 datastore = self.makeDatastore()
108 sc = self.storageClassFactory.getStorageClass("ThingOne")
109 datastore.validateConfiguration([sc])
111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo")
112 if self.validationCanFail:
113 with self.assertRaises(DatastoreValidationError):
114 datastore.validateConfiguration([sc2], logFailures=True)
116 dimensions = self.universe.extract(("visit", "physical_filter"))
117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
119 datastore.validateConfiguration([ref])
121 def testParameterValidation(self):
122 """Check that parameters are validated"""
123 sc = self.storageClassFactory.getStorageClass("ThingOne")
124 dimensions = self.universe.extract(("visit", "physical_filter"))
125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
127 datastore = self.makeDatastore()
128 data = {1: 2, 3: 4}
129 datastore.put(data, ref)
130 newdata = datastore.get(ref)
131 self.assertEqual(data, newdata)
132 with self.assertRaises(KeyError):
133 newdata = datastore.get(ref, parameters={"missing": 5})
135 def testBasicPutGet(self):
136 metrics = makeExampleMetrics()
137 datastore = self.makeDatastore()
139 # Create multiple storage classes for testing different formulations
140 storageClasses = [self.storageClassFactory.getStorageClass(sc)
141 for sc in ("StructuredData",
142 "StructuredDataJson",
143 "StructuredDataPickle")]
145 dimensions = self.universe.extract(("visit", "physical_filter"))
146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
148 for sc in storageClasses:
149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
150 print("Using storageClass: {}".format(sc.name))
151 datastore.put(metrics, ref)
153 # Does it exist?
154 self.assertTrue(datastore.exists(ref))
156 # Get
157 metricsOut = datastore.get(ref, parameters=None)
158 self.assertEqual(metrics, metricsOut)
160 uri = datastore.getURI(ref)
161 self.assertEqual(uri.scheme, self.uriScheme)
163 # Get a component -- we need to construct new refs for them
164 # with derived storage classes but with parent ID
165 for comp in ("data", "output"):
166 compRef = ref.makeComponentRef(comp)
167 output = datastore.get(compRef)
168 self.assertEqual(output, getattr(metricsOut, comp))
170 uri = datastore.getURI(compRef)
171 self.assertEqual(uri.scheme, self.uriScheme)
173 storageClass = sc
175 # Check that we can put a metric with None in a component and
176 # get it back as None
177 metricsNone = makeExampleMetrics(use_none=True)
178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"}
179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False)
180 datastore.put(metricsNone, refNone)
182 comp = "data"
183 for comp in ("data", "output"):
184 compRef = refNone.makeComponentRef(comp)
185 output = datastore.get(compRef)
186 self.assertEqual(output, getattr(metricsNone, comp))
188 # Check that a put fails if the dataset type is not supported
189 if self.hasUnsupportedPut:
190 sc = StorageClass("UnsupportedSC", pytype=type(metrics))
191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId)
192 with self.assertRaises(DatasetTypeNotSupportedError):
193 datastore.put(metrics, ref)
195 # These should raise
196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000)
197 with self.assertRaises(FileNotFoundError):
198 # non-existing file
199 datastore.get(ref)
201 # Get a URI from it
202 uri = datastore.getURI(ref, predict=True)
203 self.assertEqual(uri.scheme, self.uriScheme)
205 with self.assertRaises(FileNotFoundError):
206 datastore.getURI(ref)
208 def testCompositePutGet(self):
209 metrics = makeExampleMetrics()
210 datastore = self.makeDatastore()
212 # Create multiple storage classes for testing different formulations
213 # of composites
214 storageClasses = [self.storageClassFactory.getStorageClass(sc)
215 for sc in ("StructuredComposite",
216 "StructuredCompositeTestA",
217 "StructuredCompositeTestB")]
219 dimensions = self.universe.extract(("visit", "physical_filter"))
220 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"}
222 for sc in storageClasses:
223 print("Using storageClass: {}".format(sc.name))
224 ref = self.makeDatasetRef("metric", dimensions, sc, dataId,
225 conform=False)
227 components = sc.assembler().disassemble(metrics)
228 self.assertTrue(components)
230 compsRead = {}
231 for compName, compInfo in components.items():
232 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions,
233 components[compName].storageClass, dataId,
234 conform=False)
236 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name))
237 datastore.put(compInfo.component, compRef)
239 uri = datastore.getURI(compRef)
240 self.assertEqual(uri.scheme, self.uriScheme)
242 compsRead[compName] = datastore.get(compRef)
244 # We can generate identical files for each storage class
245 # so remove the component here
246 datastore.remove(compRef)
248 # combine all the components we read back into a new composite
249 metricsOut = sc.assembler().assemble(compsRead)
250 self.assertEqual(metrics, metricsOut)
252 def testRemove(self):
253 metrics = makeExampleMetrics()
254 datastore = self.makeDatastore()
255 # Put
256 dimensions = self.universe.extract(("visit", "physical_filter"))
257 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"}
259 sc = self.storageClassFactory.getStorageClass("StructuredData")
260 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
261 datastore.put(metrics, ref)
263 # Does it exist?
264 self.assertTrue(datastore.exists(ref))
266 # Get
267 metricsOut = datastore.get(ref)
268 self.assertEqual(metrics, metricsOut)
269 # Remove
270 datastore.remove(ref)
272 # Does it exist?
273 self.assertFalse(datastore.exists(ref))
275 # Do we now get a predicted URI?
276 uri = datastore.getURI(ref, predict=True)
277 self.assertEqual(uri.fragment, "predicted")
279 # Get should now fail
280 with self.assertRaises(FileNotFoundError):
281 datastore.get(ref)
282 # Can only delete once
283 with self.assertRaises(FileNotFoundError):
284 datastore.remove(ref)
286 def testTransfer(self):
287 metrics = makeExampleMetrics()
289 dimensions = self.universe.extract(("visit", "physical_filter"))
290 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"}
292 sc = self.storageClassFactory.getStorageClass("StructuredData")
293 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
295 inputDatastore = self.makeDatastore("test_input_datastore")
296 outputDatastore = self.makeDatastore("test_output_datastore")
298 inputDatastore.put(metrics, ref)
299 outputDatastore.transfer(inputDatastore, ref)
301 metricsOut = outputDatastore.get(ref)
302 self.assertEqual(metrics, metricsOut)
304 def testBasicTransaction(self):
305 datastore = self.makeDatastore()
306 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
307 dimensions = self.universe.extract(("visit", "physical_filter"))
308 nDatasets = 6
309 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)]
310 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False),
311 makeExampleMetrics(),)
312 for dataId in dataIds]
313 succeed = data[:nDatasets//2]
314 fail = data[nDatasets//2:]
315 # All datasets added in this transaction should continue to exist
316 with datastore.transaction():
317 for ref, metrics in succeed:
318 datastore.put(metrics, ref)
319 # Whereas datasets added in this transaction should not
320 with self.assertRaises(TransactionTestError):
321 with datastore.transaction():
322 for ref, metrics in fail:
323 datastore.put(metrics, ref)
324 raise TransactionTestError("This should propagate out of the context manager")
325 # Check for datasets that should exist
326 for ref, metrics in succeed:
327 # Does it exist?
328 self.assertTrue(datastore.exists(ref))
329 # Get
330 metricsOut = datastore.get(ref, parameters=None)
331 self.assertEqual(metrics, metricsOut)
332 # URI
333 uri = datastore.getURI(ref)
334 self.assertEqual(uri.scheme, self.uriScheme)
335 # Check for datasets that should not exist
336 for ref, _ in fail:
337 # These should raise
338 with self.assertRaises(FileNotFoundError):
339 # non-existing file
340 datastore.get(ref)
341 with self.assertRaises(FileNotFoundError):
342 datastore.getURI(ref)
344 def testNestedTransaction(self):
345 datastore = self.makeDatastore()
346 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
347 dimensions = self.universe.extract(("visit", "physical_filter"))
348 metrics = makeExampleMetrics()
350 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
351 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
352 conform=False)
353 datastore.put(metrics, refBefore)
354 with self.assertRaises(TransactionTestError):
355 with datastore.transaction():
356 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"}
357 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
358 conform=False)
359 datastore.put(metrics, refOuter)
360 with datastore.transaction():
361 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"}
362 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
363 conform=False)
364 datastore.put(metrics, refInner)
365 # All datasets should exist
366 for ref in (refBefore, refOuter, refInner):
367 metricsOut = datastore.get(ref, parameters=None)
368 self.assertEqual(metrics, metricsOut)
369 raise TransactionTestError("This should roll back the transaction")
370 # Dataset(s) inserted before the transaction should still exist
371 metricsOut = datastore.get(refBefore, parameters=None)
372 self.assertEqual(metrics, metricsOut)
373 # But all datasets inserted during the (rolled back) transaction
374 # should be gone
375 with self.assertRaises(FileNotFoundError):
376 datastore.get(refOuter)
377 with self.assertRaises(FileNotFoundError):
378 datastore.get(refInner)
380 def _prepareIngestTest(self):
381 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
382 dimensions = self.universe.extract(("visit", "physical_filter"))
383 metrics = makeExampleMetrics()
384 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
385 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
386 return metrics, ref
388 def runIngestTest(self, func, expectOutput=True):
389 metrics, ref = self._prepareIngestTest()
390 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path:
391 with open(path, 'w') as fd:
392 yaml.dump(metrics._asdict(), stream=fd)
393 func(metrics, path, ref)
395 def testIngestNoTransfer(self):
396 """Test ingesting existing files with no transfer.
397 """
398 for mode in (None, "auto"):
400 # Some datastores have auto but can't do in place transfer
401 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto:
402 continue
404 with self.subTest(mode=mode):
405 datastore = self.makeDatastore()
407 def succeed(obj, path, ref):
408 """Ingest a file already in the datastore root."""
409 # first move it into the root, and adjust the path
410 # accordingly
411 path = shutil.copy(path, datastore.root)
412 path = os.path.relpath(path, start=datastore.root)
413 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
414 self.assertEqual(obj, datastore.get(ref))
416 def failInputDoesNotExist(obj, path, ref):
417 """Can't ingest files if we're given a bad path."""
418 with self.assertRaises(FileNotFoundError):
419 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref),
420 transfer=mode)
421 self.assertFalse(datastore.exists(ref))
423 def failOutsideRoot(obj, path, ref):
424 """Can't ingest files outside of datastore root unless
425 auto."""
426 if mode == "auto":
427 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
428 self.assertTrue(datastore.exists(ref))
429 else:
430 with self.assertRaises(RuntimeError):
431 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
432 self.assertFalse(datastore.exists(ref))
434 def failNotImplemented(obj, path, ref):
435 with self.assertRaises(NotImplementedError):
436 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
438 if mode in self.ingestTransferModes:
439 self.runIngestTest(failOutsideRoot)
440 self.runIngestTest(failInputDoesNotExist)
441 self.runIngestTest(succeed)
442 else:
443 self.runIngestTest(failNotImplemented)
445 def testIngestTransfer(self):
446 """Test ingesting existing files after transferring them.
447 """
448 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"):
449 with self.subTest(mode=mode):
450 datastore = self.makeDatastore(mode)
452 def succeed(obj, path, ref):
453 """Ingest a file by transferring it to the template
454 location."""
455 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
456 self.assertEqual(obj, datastore.get(ref))
458 def failInputDoesNotExist(obj, path, ref):
459 """Can't ingest files if we're given a bad path."""
460 with self.assertRaises(FileNotFoundError):
461 # Ensure the file does not look like it is in
462 # datastore for auto mode
463 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref),
464 transfer=mode)
465 self.assertFalse(datastore.exists(ref))
467 def failOutputExists(obj, path, ref):
468 """Can't ingest files if transfer destination already
469 exists."""
470 with self.assertRaises(FileExistsError):
471 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
472 self.assertFalse(datastore.exists(ref))
474 def failNotImplemented(obj, path, ref):
475 with self.assertRaises(NotImplementedError):
476 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
478 if mode in self.ingestTransferModes:
479 self.runIngestTest(failInputDoesNotExist)
480 self.runIngestTest(succeed, expectOutput=(mode != "move"))
481 self.runIngestTest(failOutputExists)
482 else:
483 self.runIngestTest(failNotImplemented)
485 def testIngestSymlinkOfSymlink(self):
486 """Special test for symlink to a symlink ingest"""
487 metrics, ref = self._prepareIngestTest()
488 # The aim of this test is to create a dataset on disk, then
489 # create a symlink to it and finally ingest the symlink such that
490 # the symlink in the datastore points to the original dataset.
491 for mode in ("symlink", "relsymlink"):
492 if mode not in self.ingestTransferModes:
493 continue
495 print(f"Trying mode {mode}")
496 with lsst.utils.tests.getTempFilePath(".yaml") as realpath:
497 with open(realpath, 'w') as fd:
498 yaml.dump(metrics._asdict(), stream=fd)
499 with lsst.utils.tests.getTempFilePath(".yaml") as sympath:
500 os.symlink(os.path.abspath(realpath), sympath)
502 datastore = self.makeDatastore()
503 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode)
505 uri = datastore.getURI(ref)
506 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}")
507 self.assertTrue(os.path.islink(uri.path))
509 linkTarget = os.readlink(uri.path)
510 if mode == "relsymlink":
511 self.assertFalse(os.path.isabs(linkTarget))
512 else:
513 self.assertEqual(linkTarget, os.path.abspath(realpath))
515 # Check that we can get the dataset back regardless of mode
516 metric2 = datastore.get(ref)
517 self.assertEqual(metric2, metrics)
519 # Cleanup the file for next time round loop
520 # since it will get the same file name in store
521 datastore.remove(ref)
524class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase):
525 """PosixDatastore specialization"""
526 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
527 uriScheme = "file"
528 canIngestNoTransferAuto = True
529 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto")
530 isEphemeral = False
531 rootKeys = ("root",)
532 validationCanFail = True
534 def setUp(self):
535 # Override the working directory before calling the base class
536 self.root = tempfile.mkdtemp(dir=TESTDIR)
537 super().setUp()
540class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase):
541 """Posix datastore tests but with checksums disabled."""
542 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml")
544 def testChecksum(self):
545 """Ensure that checksums have not been calculated."""
547 datastore = self.makeDatastore()
548 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
549 dimensions = self.universe.extract(("visit", "physical_filter"))
550 metrics = makeExampleMetrics()
552 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
553 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
554 conform=False)
556 # Configuration should have disabled checksum calculation
557 datastore.put(metrics, ref)
558 info = datastore.getStoredItemInfo(ref)
559 self.assertIsNone(info.checksum)
561 # Remove put back but with checksums enabled explicitly
562 datastore.remove(ref)
563 datastore.useChecksum = True
564 datastore.put(metrics, ref)
566 info = datastore.getStoredItemInfo(ref)
567 self.assertIsNotNone(info.checksum)
570class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase):
571 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
573 def setUp(self):
574 # Override the working directory before calling the base class
575 self.root = tempfile.mkdtemp(dir=TESTDIR)
576 super().setUp()
578 def testCleanup(self):
579 """Test that a failed formatter write does cleanup a partial file."""
580 metrics = makeExampleMetrics()
581 datastore = self.makeDatastore()
583 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
585 dimensions = self.universe.extract(("visit", "physical_filter"))
586 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
588 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
590 # Determine where the file will end up (we assume Formatters use
591 # the same file extension)
592 expectedUri = datastore.getURI(ref, predict=True)
593 self.assertEqual(expectedUri.fragment, "predicted")
595 expectedFile = expectedUri.path
596 self.assertTrue(expectedFile.endswith(".yaml"),
597 f"Is there a file extension in {expectedUri}")
599 # Try formatter that fails and formatter that fails and leaves
600 # a file behind
601 for formatter in (BadWriteFormatter, BadNoWriteFormatter):
602 with self.subTest(formatter=formatter):
604 # Monkey patch the formatter
605 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter,
606 overwrite=True)
608 # Try to put the dataset, it should fail
609 with self.assertRaises(Exception):
610 datastore.put(metrics, ref)
612 # Check that there is no file on disk
613 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
615 # Check that there is a directory
616 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)),
617 f"Check for existence of directory {os.path.dirname(expectedFile)}")
619 # Force YamlFormatter and check that this time a file is written
620 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter,
621 overwrite=True)
622 datastore.put(metrics, ref)
623 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
624 datastore.remove(ref)
625 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}")
628class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase):
629 """PosixDatastore specialization"""
630 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml")
631 uriScheme = "mem"
632 hasUnsupportedPut = False
633 ingestTransferModes = ()
634 isEphemeral = True
635 rootKeys = None
636 validationCanFail = False
639class ChainedDatastoreTestCase(PosixDatastoreTestCase):
640 """ChainedDatastore specialization using a POSIXDatastore"""
641 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
642 hasUnsupportedPut = False
643 canIngestNoTransferAuto = False
644 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto")
645 isEphemeral = False
646 rootKeys = (".datastores.1.root", ".datastores.2.root")
647 validationCanFail = True
650class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase):
651 """ChainedDatastore specialization using all InMemoryDatastore"""
652 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml")
653 validationCanFail = False
656class DatastoreConstraintsTests(DatastoreTestsBase):
657 """Basic tests of constraints model of Datastores."""
659 def testConstraints(self):
660 """Test constraints model. Assumes that each test class has the
661 same constraints."""
662 metrics = makeExampleMetrics()
663 datastore = self.makeDatastore()
665 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
666 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
667 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
668 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
670 # Write empty file suitable for ingest check (JSON and YAML variants)
671 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
672 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
673 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False),
674 ("metric33", sc1, True), ("metric2", sc2, True)):
675 # Choose different temp file depending on StorageClass
676 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
678 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name):
679 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False)
680 if accepted:
681 datastore.put(metrics, ref)
682 self.assertTrue(datastore.exists(ref))
683 datastore.remove(ref)
685 # Try ingest
686 if self.canIngest:
687 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
688 self.assertTrue(datastore.exists(ref))
689 datastore.remove(ref)
690 else:
691 with self.assertRaises(DatasetTypeNotSupportedError):
692 datastore.put(metrics, ref)
693 self.assertFalse(datastore.exists(ref))
695 # Again with ingest
696 if self.canIngest:
697 with self.assertRaises(DatasetTypeNotSupportedError):
698 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
699 self.assertFalse(datastore.exists(ref))
702class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
703 """PosixDatastore specialization"""
704 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml")
705 canIngest = True
707 def setUp(self):
708 # Override the working directory before calling the base class
709 self.root = tempfile.mkdtemp(dir=TESTDIR)
710 super().setUp()
713class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
714 """InMemoryDatastore specialization"""
715 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml")
716 canIngest = False
719class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase):
720 """ChainedDatastore specialization using a POSIXDatastore and constraints
721 at the ChainedDatstore """
722 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml")
725class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase):
726 """ChainedDatastore specialization using a POSIXDatastore"""
727 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml")
730class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase):
731 """ChainedDatastore specialization using all InMemoryDatastore"""
732 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml")
733 canIngest = False
736class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase):
737 """Test that a chained datastore can control constraints per-datastore
738 even if child datastore would accept."""
740 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml")
742 def setUp(self):
743 # Override the working directory before calling the base class
744 self.root = tempfile.mkdtemp(dir=TESTDIR)
745 super().setUp()
747 def testConstraints(self):
748 """Test chained datastore constraints model."""
749 metrics = makeExampleMetrics()
750 datastore = self.makeDatastore()
752 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
753 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
754 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
755 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
756 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"}
758 # Write empty file suitable for ingest check (JSON and YAML variants)
759 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
760 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
762 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True),
763 ("metric2", dataId1, sc1, (False, False, False), False),
764 ("metric2", dataId2, sc1, (True, False, False), False),
765 ("metric33", dataId2, sc2, (True, True, False), True),
766 ("metric2", dataId1, sc2, (False, True, False), True)):
768 # Choose different temp file depending on StorageClass
769 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
771 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name):
772 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId,
773 conform=False)
774 if any(accept):
775 datastore.put(metrics, ref)
776 self.assertTrue(datastore.exists(ref))
778 # Check each datastore inside the chained datastore
779 for childDatastore, expected in zip(datastore.datastores, accept):
780 self.assertEqual(childDatastore.exists(ref), expected,
781 f"Testing presence of {ref} in datastore {childDatastore.name}")
783 datastore.remove(ref)
785 # Check that ingest works
786 if ingest:
787 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
788 self.assertTrue(datastore.exists(ref))
790 # Check each datastore inside the chained datastore
791 for childDatastore, expected in zip(datastore.datastores, accept):
792 # Ephemeral datastores means InMemory at the moment
793 # and that does not accept ingest of files.
794 if childDatastore.isEphemeral:
795 expected = False
796 self.assertEqual(childDatastore.exists(ref), expected,
797 f"Testing presence of ingested {ref} in datastore"
798 f" {childDatastore.name}")
800 datastore.remove(ref)
801 else:
802 with self.assertRaises(DatasetTypeNotSupportedError):
803 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
805 else:
806 with self.assertRaises(DatasetTypeNotSupportedError):
807 datastore.put(metrics, ref)
808 self.assertFalse(datastore.exists(ref))
810 # Again with ingest
811 with self.assertRaises(DatasetTypeNotSupportedError):
812 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
813 self.assertFalse(datastore.exists(ref))
816if __name__ == "__main__": 816 ↛ 817line 816 didn't jump to line 817, because the condition on line 816 was never true
817 unittest.main()