Coverage for tests/test_datastore.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import unittest
24import shutil
25import yaml
26import tempfile
27import lsst.utils.tests
29from lsst.utils import doImport
31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset
32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError
33from lsst.daf.butler.formatters.yaml import YamlFormatter
35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter,
36 BadNoWriteFormatter, MetricsExample, DummyRegistry)
39TESTDIR = os.path.dirname(__file__)
42def makeExampleMetrics(use_none=False):
43 if use_none:
44 array = None
45 else:
46 array = [563, 234, 456.7, 105, 2054, -1045]
47 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
48 {"a": [1, 2, 3],
49 "b": {"blue": 5, "red": "green"}},
50 array,
51 )
54class TransactionTestError(Exception):
55 """Specific error for transactions, to prevent misdiagnosing
56 that might otherwise occur when a standard exception is used.
57 """
58 pass
61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper):
62 """Support routines for datastore testing"""
63 root = None
65 @classmethod
66 def setUpClass(cls):
67 # Storage Classes are fixed for all datastores in these tests
68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
69 cls.storageClassFactory = StorageClassFactory()
70 cls.storageClassFactory.addFromConfig(scConfigFile)
72 # Read the Datastore config so we can get the class
73 # information (since we should not assume the constructor
74 # name here, but rely on the configuration file itself)
75 datastoreConfig = DatastoreConfig(cls.configFile)
76 cls.datastoreType = doImport(datastoreConfig["cls"])
77 cls.universe = DimensionUniverse()
79 def setUp(self):
80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig)
82 def tearDown(self):
83 if self.root is not None and os.path.exists(self.root):
84 shutil.rmtree(self.root, ignore_errors=True)
87class DatastoreTests(DatastoreTestsBase):
88 """Some basic tests of a simple datastore."""
90 hasUnsupportedPut = True
92 def testConfigRoot(self):
93 full = DatastoreConfig(self.configFile)
94 config = DatastoreConfig(self.configFile, mergeDefaults=False)
95 newroot = "/random/location"
96 self.datastoreType.setConfigRoot(newroot, config, full)
97 if self.rootKeys:
98 for k in self.rootKeys:
99 self.assertIn(newroot, config[k])
101 def testConstructor(self):
102 datastore = self.makeDatastore()
103 self.assertIsNotNone(datastore)
104 self.assertIs(datastore.isEphemeral, self.isEphemeral)
106 def testConfigurationValidation(self):
107 datastore = self.makeDatastore()
108 sc = self.storageClassFactory.getStorageClass("ThingOne")
109 datastore.validateConfiguration([sc])
111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo")
112 if self.validationCanFail:
113 with self.assertRaises(DatastoreValidationError):
114 datastore.validateConfiguration([sc2], logFailures=True)
116 dimensions = self.universe.extract(("visit", "physical_filter"))
117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
119 datastore.validateConfiguration([ref])
121 def testParameterValidation(self):
122 """Check that parameters are validated"""
123 sc = self.storageClassFactory.getStorageClass("ThingOne")
124 dimensions = self.universe.extract(("visit", "physical_filter"))
125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
127 datastore = self.makeDatastore()
128 data = {1: 2, 3: 4}
129 datastore.put(data, ref)
130 newdata = datastore.get(ref)
131 self.assertEqual(data, newdata)
132 with self.assertRaises(KeyError):
133 newdata = datastore.get(ref, parameters={"missing": 5})
135 def testBasicPutGet(self):
136 metrics = makeExampleMetrics()
137 datastore = self.makeDatastore()
139 # Create multiple storage classes for testing different formulations
140 storageClasses = [self.storageClassFactory.getStorageClass(sc)
141 for sc in ("StructuredData",
142 "StructuredDataJson",
143 "StructuredDataPickle")]
145 dimensions = self.universe.extract(("visit", "physical_filter"))
146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
148 for sc in storageClasses:
149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
150 print("Using storageClass: {}".format(sc.name))
151 datastore.put(metrics, ref)
153 # Does it exist?
154 self.assertTrue(datastore.exists(ref))
156 # Get
157 metricsOut = datastore.get(ref, parameters=None)
158 self.assertEqual(metrics, metricsOut)
160 uri = datastore.getURI(ref)
161 self.assertEqual(uri.scheme, self.uriScheme)
163 # Get a component -- we need to construct new refs for them
164 # with derived storage classes but with parent ID
165 for comp in ("data", "output"):
166 compRef = ref.makeComponentRef(comp)
167 output = datastore.get(compRef)
168 self.assertEqual(output, getattr(metricsOut, comp))
170 uri = datastore.getURI(compRef)
171 self.assertEqual(uri.scheme, self.uriScheme)
173 storageClass = sc
175 # Check that we can put a metric with None in a component and
176 # get it back as None
177 metricsNone = makeExampleMetrics(use_none=True)
178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"}
179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False)
180 datastore.put(metricsNone, refNone)
182 comp = "data"
183 for comp in ("data", "output"):
184 compRef = refNone.makeComponentRef(comp)
185 output = datastore.get(compRef)
186 self.assertEqual(output, getattr(metricsNone, comp))
188 # Check that a put fails if the dataset type is not supported
189 if self.hasUnsupportedPut:
190 sc = StorageClass("UnsupportedSC", pytype=type(metrics))
191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId)
192 with self.assertRaises(DatasetTypeNotSupportedError):
193 datastore.put(metrics, ref)
195 # These should raise
196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000)
197 with self.assertRaises(FileNotFoundError):
198 # non-existing file
199 datastore.get(ref)
201 # Get a URI from it
202 uri = datastore.getURI(ref, predict=True)
203 self.assertEqual(uri.scheme, self.uriScheme)
205 with self.assertRaises(FileNotFoundError):
206 datastore.getURI(ref)
208 def testDisassembly(self):
209 """Test disassembly within datastore."""
210 metrics = makeExampleMetrics()
211 if self.isEphemeral:
212 # in-memory datastore does not disassemble
213 return
215 # Create multiple storage classes for testing different formulations
216 # of composites. One of these will not disassemble to provide
217 # a reference.
218 storageClasses = [self.storageClassFactory.getStorageClass(sc)
219 for sc in ("StructuredComposite",
220 "StructuredCompositeTestA",
221 "StructuredCompositeTestB",
222 "StructuredCompositeReadComp",
223 "StructuredData", # No disassembly
224 "StructuredCompositeReadCompNoDisassembly",
225 )]
227 # Create the test datastore
228 datastore = self.makeDatastore()
230 # Dummy dataId
231 dimensions = self.universe.extract(("visit", "physical_filter"))
232 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"}
234 for i, sc in enumerate(storageClasses):
235 with self.subTest(storageClass=sc.name):
236 # Create a different dataset type each time round
237 # so that a test failure in this subtest does not trigger
238 # a cascade of tests because of file clashes
239 ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId,
240 conform=False)
242 disassembled = sc.name not in {"StructuredData", "StructuredCompositeReadCompNoDisassembly"}
244 datastore.put(metrics, ref)
246 baseURI, compURIs = datastore.getURIs(ref)
247 if disassembled:
248 self.assertIsNone(baseURI)
249 self.assertEqual(set(compURIs), {"data", "output", "summary"})
250 else:
251 self.assertIsNotNone(baseURI)
252 self.assertEqual(compURIs, {})
254 metrics_get = datastore.get(ref)
255 self.assertEqual(metrics_get, metrics)
257 # Retrieve the composite with read parameter
258 stop = 4
259 metrics_get = datastore.get(ref, parameters={"slice": slice(stop)})
260 self.assertEqual(metrics_get.summary, metrics.summary)
261 self.assertEqual(metrics_get.output, metrics.output)
262 self.assertEqual(metrics_get.data, metrics.data[:stop])
264 # Retrieve a component
265 data = datastore.get(ref.makeComponentRef("data"))
266 self.assertEqual(data, metrics.data)
268 # On supported storage classes attempt to access a read
269 # only component
270 if "ReadComp" in sc.name:
271 cRef = ref.makeComponentRef("counter")
272 counter = datastore.get(cRef)
273 self.assertEqual(counter, len(metrics.data))
275 counter = datastore.get(cRef, parameters={"slice": slice(stop)})
276 self.assertEqual(counter, stop)
278 datastore.remove(ref)
280 def testRegistryCompositePutGet(self):
281 """Tests the case where registry disassembles and puts to datastore.
282 """
283 metrics = makeExampleMetrics()
284 datastore = self.makeDatastore()
286 # Create multiple storage classes for testing different formulations
287 # of composites
288 storageClasses = [self.storageClassFactory.getStorageClass(sc)
289 for sc in ("StructuredComposite",
290 "StructuredCompositeTestA",
291 "StructuredCompositeTestB",
292 )]
294 dimensions = self.universe.extract(("visit", "physical_filter"))
295 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"}
297 for sc in storageClasses:
298 print("Using storageClass: {}".format(sc.name))
299 ref = self.makeDatasetRef("metric", dimensions, sc, dataId,
300 conform=False)
302 components = sc.assembler().disassemble(metrics)
303 self.assertTrue(components)
305 compsRead = {}
306 for compName, compInfo in components.items():
307 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions,
308 components[compName].storageClass, dataId,
309 conform=False)
311 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name))
312 datastore.put(compInfo.component, compRef)
314 uri = datastore.getURI(compRef)
315 self.assertEqual(uri.scheme, self.uriScheme)
317 compsRead[compName] = datastore.get(compRef)
319 # We can generate identical files for each storage class
320 # so remove the component here
321 datastore.remove(compRef)
323 # combine all the components we read back into a new composite
324 metricsOut = sc.assembler().assemble(compsRead)
325 self.assertEqual(metrics, metricsOut)
327 def testRemove(self):
328 metrics = makeExampleMetrics()
329 datastore = self.makeDatastore()
330 # Put
331 dimensions = self.universe.extract(("visit", "physical_filter"))
332 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"}
334 sc = self.storageClassFactory.getStorageClass("StructuredData")
335 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
336 datastore.put(metrics, ref)
338 # Does it exist?
339 self.assertTrue(datastore.exists(ref))
341 # Get
342 metricsOut = datastore.get(ref)
343 self.assertEqual(metrics, metricsOut)
344 # Remove
345 datastore.remove(ref)
347 # Does it exist?
348 self.assertFalse(datastore.exists(ref))
350 # Do we now get a predicted URI?
351 uri = datastore.getURI(ref, predict=True)
352 self.assertEqual(uri.fragment, "predicted")
354 # Get should now fail
355 with self.assertRaises(FileNotFoundError):
356 datastore.get(ref)
357 # Can only delete once
358 with self.assertRaises(FileNotFoundError):
359 datastore.remove(ref)
361 def testTransfer(self):
362 metrics = makeExampleMetrics()
364 dimensions = self.universe.extract(("visit", "physical_filter"))
365 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"}
367 sc = self.storageClassFactory.getStorageClass("StructuredData")
368 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
370 inputDatastore = self.makeDatastore("test_input_datastore")
371 outputDatastore = self.makeDatastore("test_output_datastore")
373 inputDatastore.put(metrics, ref)
374 outputDatastore.transfer(inputDatastore, ref)
376 metricsOut = outputDatastore.get(ref)
377 self.assertEqual(metrics, metricsOut)
379 def testBasicTransaction(self):
380 datastore = self.makeDatastore()
381 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
382 dimensions = self.universe.extract(("visit", "physical_filter"))
383 nDatasets = 6
384 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)]
385 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False),
386 makeExampleMetrics(),)
387 for dataId in dataIds]
388 succeed = data[:nDatasets//2]
389 fail = data[nDatasets//2:]
390 # All datasets added in this transaction should continue to exist
391 with datastore.transaction():
392 for ref, metrics in succeed:
393 datastore.put(metrics, ref)
394 # Whereas datasets added in this transaction should not
395 with self.assertRaises(TransactionTestError):
396 with datastore.transaction():
397 for ref, metrics in fail:
398 datastore.put(metrics, ref)
399 raise TransactionTestError("This should propagate out of the context manager")
400 # Check for datasets that should exist
401 for ref, metrics in succeed:
402 # Does it exist?
403 self.assertTrue(datastore.exists(ref))
404 # Get
405 metricsOut = datastore.get(ref, parameters=None)
406 self.assertEqual(metrics, metricsOut)
407 # URI
408 uri = datastore.getURI(ref)
409 self.assertEqual(uri.scheme, self.uriScheme)
410 # Check for datasets that should not exist
411 for ref, _ in fail:
412 # These should raise
413 with self.assertRaises(FileNotFoundError):
414 # non-existing file
415 datastore.get(ref)
416 with self.assertRaises(FileNotFoundError):
417 datastore.getURI(ref)
419 def testNestedTransaction(self):
420 datastore = self.makeDatastore()
421 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
422 dimensions = self.universe.extract(("visit", "physical_filter"))
423 metrics = makeExampleMetrics()
425 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
426 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
427 conform=False)
428 datastore.put(metrics, refBefore)
429 with self.assertRaises(TransactionTestError):
430 with datastore.transaction():
431 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"}
432 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
433 conform=False)
434 datastore.put(metrics, refOuter)
435 with datastore.transaction():
436 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"}
437 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
438 conform=False)
439 datastore.put(metrics, refInner)
440 # All datasets should exist
441 for ref in (refBefore, refOuter, refInner):
442 metricsOut = datastore.get(ref, parameters=None)
443 self.assertEqual(metrics, metricsOut)
444 raise TransactionTestError("This should roll back the transaction")
445 # Dataset(s) inserted before the transaction should still exist
446 metricsOut = datastore.get(refBefore, parameters=None)
447 self.assertEqual(metrics, metricsOut)
448 # But all datasets inserted during the (rolled back) transaction
449 # should be gone
450 with self.assertRaises(FileNotFoundError):
451 datastore.get(refOuter)
452 with self.assertRaises(FileNotFoundError):
453 datastore.get(refInner)
455 def _prepareIngestTest(self):
456 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
457 dimensions = self.universe.extract(("visit", "physical_filter"))
458 metrics = makeExampleMetrics()
459 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
460 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
461 return metrics, ref
463 def runIngestTest(self, func, expectOutput=True):
464 metrics, ref = self._prepareIngestTest()
465 # The file will be deleted after the test.
466 # For symlink tests this leads to a situation where the datastore
467 # points to a file that does not exist. This will make os.path.exist
468 # return False but then the new symlink will fail with
469 # FileExistsError later in the code so the test still passes.
470 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path:
471 with open(path, 'w') as fd:
472 yaml.dump(metrics._asdict(), stream=fd)
473 func(metrics, path, ref)
475 def testIngestNoTransfer(self):
476 """Test ingesting existing files with no transfer.
477 """
478 for mode in (None, "auto"):
480 # Some datastores have auto but can't do in place transfer
481 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto:
482 continue
484 with self.subTest(mode=mode):
485 datastore = self.makeDatastore()
487 def succeed(obj, path, ref):
488 """Ingest a file already in the datastore root."""
489 # first move it into the root, and adjust the path
490 # accordingly
491 path = shutil.copy(path, datastore.root)
492 path = os.path.relpath(path, start=datastore.root)
493 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
494 self.assertEqual(obj, datastore.get(ref))
496 def failInputDoesNotExist(obj, path, ref):
497 """Can't ingest files if we're given a bad path."""
498 with self.assertRaises(FileNotFoundError):
499 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref),
500 transfer=mode)
501 self.assertFalse(datastore.exists(ref))
503 def failOutsideRoot(obj, path, ref):
504 """Can't ingest files outside of datastore root unless
505 auto."""
506 if mode == "auto":
507 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
508 self.assertTrue(datastore.exists(ref))
509 else:
510 with self.assertRaises(RuntimeError):
511 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
512 self.assertFalse(datastore.exists(ref))
514 def failNotImplemented(obj, path, ref):
515 with self.assertRaises(NotImplementedError):
516 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
518 if mode in self.ingestTransferModes:
519 self.runIngestTest(failOutsideRoot)
520 self.runIngestTest(failInputDoesNotExist)
521 self.runIngestTest(succeed)
522 else:
523 self.runIngestTest(failNotImplemented)
525 def testIngestTransfer(self):
526 """Test ingesting existing files after transferring them.
527 """
528 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"):
529 with self.subTest(mode=mode):
530 datastore = self.makeDatastore(mode)
532 def succeed(obj, path, ref):
533 """Ingest a file by transferring it to the template
534 location."""
535 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
536 self.assertEqual(obj, datastore.get(ref))
538 def failInputDoesNotExist(obj, path, ref):
539 """Can't ingest files if we're given a bad path."""
540 with self.assertRaises(FileNotFoundError):
541 # Ensure the file does not look like it is in
542 # datastore for auto mode
543 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref),
544 transfer=mode)
545 self.assertFalse(datastore.exists(ref), f"Checking not in datastore using mode {mode}")
547 def failOutputExists(obj, path, ref):
548 """Can't ingest files if transfer destination already
549 exists."""
550 with self.assertRaises(FileExistsError):
551 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
552 self.assertFalse(datastore.exists(ref), f"Checking not in datastore using mode {mode}")
554 def failNotImplemented(obj, path, ref):
555 with self.assertRaises(NotImplementedError):
556 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
558 if mode in self.ingestTransferModes:
559 self.runIngestTest(failInputDoesNotExist)
560 self.runIngestTest(succeed, expectOutput=(mode != "move"))
561 self.runIngestTest(failOutputExists)
562 else:
563 self.runIngestTest(failNotImplemented)
565 def testIngestSymlinkOfSymlink(self):
566 """Special test for symlink to a symlink ingest"""
567 metrics, ref = self._prepareIngestTest()
568 # The aim of this test is to create a dataset on disk, then
569 # create a symlink to it and finally ingest the symlink such that
570 # the symlink in the datastore points to the original dataset.
571 for mode in ("symlink", "relsymlink"):
572 if mode not in self.ingestTransferModes:
573 continue
575 print(f"Trying mode {mode}")
576 with lsst.utils.tests.getTempFilePath(".yaml") as realpath:
577 with open(realpath, 'w') as fd:
578 yaml.dump(metrics._asdict(), stream=fd)
579 with lsst.utils.tests.getTempFilePath(".yaml") as sympath:
580 os.symlink(os.path.abspath(realpath), sympath)
582 datastore = self.makeDatastore()
583 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode)
585 uri = datastore.getURI(ref)
586 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}")
587 self.assertTrue(os.path.islink(uri.path))
589 linkTarget = os.readlink(uri.path)
590 if mode == "relsymlink":
591 self.assertFalse(os.path.isabs(linkTarget))
592 else:
593 self.assertEqual(linkTarget, os.path.abspath(realpath))
595 # Check that we can get the dataset back regardless of mode
596 metric2 = datastore.get(ref)
597 self.assertEqual(metric2, metrics)
599 # Cleanup the file for next time round loop
600 # since it will get the same file name in store
601 datastore.remove(ref)
604class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase):
605 """PosixDatastore specialization"""
606 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
607 uriScheme = "file"
608 canIngestNoTransferAuto = True
609 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto")
610 isEphemeral = False
611 rootKeys = ("root",)
612 validationCanFail = True
614 def setUp(self):
615 # Override the working directory before calling the base class
616 self.root = tempfile.mkdtemp(dir=TESTDIR)
617 super().setUp()
620class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase):
621 """Posix datastore tests but with checksums disabled."""
622 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml")
624 def testChecksum(self):
625 """Ensure that checksums have not been calculated."""
627 datastore = self.makeDatastore()
628 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
629 dimensions = self.universe.extract(("visit", "physical_filter"))
630 metrics = makeExampleMetrics()
632 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
633 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
634 conform=False)
636 # Configuration should have disabled checksum calculation
637 datastore.put(metrics, ref)
638 infos = datastore.getStoredItemsInfo(ref)
639 self.assertIsNone(infos[0].checksum)
641 # Remove put back but with checksums enabled explicitly
642 datastore.remove(ref)
643 datastore.useChecksum = True
644 datastore.put(metrics, ref)
646 infos = datastore.getStoredItemsInfo(ref)
647 self.assertIsNotNone(infos[0].checksum)
650class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase):
651 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
653 def setUp(self):
654 # Override the working directory before calling the base class
655 self.root = tempfile.mkdtemp(dir=TESTDIR)
656 super().setUp()
658 def testCleanup(self):
659 """Test that a failed formatter write does cleanup a partial file."""
660 metrics = makeExampleMetrics()
661 datastore = self.makeDatastore()
663 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
665 dimensions = self.universe.extract(("visit", "physical_filter"))
666 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
668 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
670 # Determine where the file will end up (we assume Formatters use
671 # the same file extension)
672 expectedUri = datastore.getURI(ref, predict=True)
673 self.assertEqual(expectedUri.fragment, "predicted")
675 expectedFile = expectedUri.path
676 self.assertTrue(expectedFile.endswith(".yaml"),
677 f"Is there a file extension in {expectedUri}")
679 # Try formatter that fails and formatter that fails and leaves
680 # a file behind
681 for formatter in (BadWriteFormatter, BadNoWriteFormatter):
682 with self.subTest(formatter=formatter):
684 # Monkey patch the formatter
685 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter,
686 overwrite=True)
688 # Try to put the dataset, it should fail
689 with self.assertRaises(Exception):
690 datastore.put(metrics, ref)
692 # Check that there is no file on disk
693 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
695 # Check that there is a directory
696 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)),
697 f"Check for existence of directory {os.path.dirname(expectedFile)}")
699 # Force YamlFormatter and check that this time a file is written
700 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter,
701 overwrite=True)
702 datastore.put(metrics, ref)
703 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
704 datastore.remove(ref)
705 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}")
708class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase):
709 """PosixDatastore specialization"""
710 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml")
711 uriScheme = "mem"
712 hasUnsupportedPut = False
713 ingestTransferModes = ()
714 isEphemeral = True
715 rootKeys = None
716 validationCanFail = False
719class ChainedDatastoreTestCase(PosixDatastoreTestCase):
720 """ChainedDatastore specialization using a POSIXDatastore"""
721 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
722 hasUnsupportedPut = False
723 canIngestNoTransferAuto = False
724 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto")
725 isEphemeral = False
726 rootKeys = (".datastores.1.root", ".datastores.2.root")
727 validationCanFail = True
730class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase):
731 """ChainedDatastore specialization using all InMemoryDatastore"""
732 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml")
733 validationCanFail = False
736class DatastoreConstraintsTests(DatastoreTestsBase):
737 """Basic tests of constraints model of Datastores."""
739 def testConstraints(self):
740 """Test constraints model. Assumes that each test class has the
741 same constraints."""
742 metrics = makeExampleMetrics()
743 datastore = self.makeDatastore()
745 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
746 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
747 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
748 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
750 # Write empty file suitable for ingest check (JSON and YAML variants)
751 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
752 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
753 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False),
754 ("metric33", sc1, True), ("metric2", sc2, True)):
755 # Choose different temp file depending on StorageClass
756 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
758 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name):
759 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False)
760 if accepted:
761 datastore.put(metrics, ref)
762 self.assertTrue(datastore.exists(ref))
763 datastore.remove(ref)
765 # Try ingest
766 if self.canIngest:
767 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
768 self.assertTrue(datastore.exists(ref))
769 datastore.remove(ref)
770 else:
771 with self.assertRaises(DatasetTypeNotSupportedError):
772 datastore.put(metrics, ref)
773 self.assertFalse(datastore.exists(ref))
775 # Again with ingest
776 if self.canIngest:
777 with self.assertRaises(DatasetTypeNotSupportedError):
778 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
779 self.assertFalse(datastore.exists(ref))
782class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
783 """PosixDatastore specialization"""
784 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml")
785 canIngest = True
787 def setUp(self):
788 # Override the working directory before calling the base class
789 self.root = tempfile.mkdtemp(dir=TESTDIR)
790 super().setUp()
793class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
794 """InMemoryDatastore specialization"""
795 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml")
796 canIngest = False
799class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase):
800 """ChainedDatastore specialization using a POSIXDatastore and constraints
801 at the ChainedDatstore """
802 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml")
805class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase):
806 """ChainedDatastore specialization using a POSIXDatastore"""
807 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml")
810class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase):
811 """ChainedDatastore specialization using all InMemoryDatastore"""
812 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml")
813 canIngest = False
816class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase):
817 """Test that a chained datastore can control constraints per-datastore
818 even if child datastore would accept."""
820 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml")
822 def setUp(self):
823 # Override the working directory before calling the base class
824 self.root = tempfile.mkdtemp(dir=TESTDIR)
825 super().setUp()
827 def testConstraints(self):
828 """Test chained datastore constraints model."""
829 metrics = makeExampleMetrics()
830 datastore = self.makeDatastore()
832 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
833 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
834 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
835 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
836 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"}
838 # Write empty file suitable for ingest check (JSON and YAML variants)
839 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
840 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
842 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True),
843 ("metric2", dataId1, sc1, (False, False, False), False),
844 ("metric2", dataId2, sc1, (True, False, False), False),
845 ("metric33", dataId2, sc2, (True, True, False), True),
846 ("metric2", dataId1, sc2, (False, True, False), True)):
848 # Choose different temp file depending on StorageClass
849 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
851 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name):
852 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId,
853 conform=False)
854 if any(accept):
855 datastore.put(metrics, ref)
856 self.assertTrue(datastore.exists(ref))
858 # Check each datastore inside the chained datastore
859 for childDatastore, expected in zip(datastore.datastores, accept):
860 self.assertEqual(childDatastore.exists(ref), expected,
861 f"Testing presence of {ref} in datastore {childDatastore.name}")
863 datastore.remove(ref)
865 # Check that ingest works
866 if ingest:
867 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
868 self.assertTrue(datastore.exists(ref))
870 # Check each datastore inside the chained datastore
871 for childDatastore, expected in zip(datastore.datastores, accept):
872 # Ephemeral datastores means InMemory at the moment
873 # and that does not accept ingest of files.
874 if childDatastore.isEphemeral:
875 expected = False
876 self.assertEqual(childDatastore.exists(ref), expected,
877 f"Testing presence of ingested {ref} in datastore"
878 f" {childDatastore.name}")
880 datastore.remove(ref)
881 else:
882 with self.assertRaises(DatasetTypeNotSupportedError):
883 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
885 else:
886 with self.assertRaises(DatasetTypeNotSupportedError):
887 datastore.put(metrics, ref)
888 self.assertFalse(datastore.exists(ref))
890 # Again with ingest
891 with self.assertRaises(DatasetTypeNotSupportedError):
892 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
893 self.assertFalse(datastore.exists(ref))
896if __name__ == "__main__": 896 ↛ 897line 896 didn't jump to line 897, because the condition on line 896 was never true
897 unittest.main()