Coverage for tests/test_datastore.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import unittest
24import shutil
25import yaml
26import tempfile
27import lsst.utils.tests
29from lsst.utils import doImport
31from lsst.daf.butler import StorageClassFactory, StorageClass, DimensionUniverse, FileDataset
32from lsst.daf.butler import DatastoreConfig, DatasetTypeNotSupportedError, DatastoreValidationError
33from lsst.daf.butler.formatters.yaml import YamlFormatter
35from lsst.daf.butler.tests import (DatasetTestHelper, DatastoreTestHelper, BadWriteFormatter,
36 BadNoWriteFormatter, MetricsExample, DummyRegistry)
39TESTDIR = os.path.dirname(__file__)
42def makeExampleMetrics(use_none=False):
43 if use_none:
44 array = None
45 else:
46 array = [563, 234, 456.7, 105, 2054, -1045]
47 return MetricsExample({"AM1": 5.2, "AM2": 30.6},
48 {"a": [1, 2, 3],
49 "b": {"blue": 5, "red": "green"}},
50 array,
51 )
54class TransactionTestError(Exception):
55 """Specific error for transactions, to prevent misdiagnosing
56 that might otherwise occur when a standard exception is used.
57 """
58 pass
61class DatastoreTestsBase(DatasetTestHelper, DatastoreTestHelper):
62 """Support routines for datastore testing"""
63 root = None
65 @classmethod
66 def setUpClass(cls):
67 # Storage Classes are fixed for all datastores in these tests
68 scConfigFile = os.path.join(TESTDIR, "config/basic/storageClasses.yaml")
69 cls.storageClassFactory = StorageClassFactory()
70 cls.storageClassFactory.addFromConfig(scConfigFile)
72 # Read the Datastore config so we can get the class
73 # information (since we should not assume the constructor
74 # name here, but rely on the configuration file itself)
75 datastoreConfig = DatastoreConfig(cls.configFile)
76 cls.datastoreType = doImport(datastoreConfig["cls"])
77 cls.universe = DimensionUniverse()
79 def setUp(self):
80 self.setUpDatastoreTests(DummyRegistry, DatastoreConfig)
82 def tearDown(self):
83 if self.root is not None and os.path.exists(self.root):
84 shutil.rmtree(self.root, ignore_errors=True)
87class DatastoreTests(DatastoreTestsBase):
88 """Some basic tests of a simple datastore."""
90 hasUnsupportedPut = True
92 def testConfigRoot(self):
93 full = DatastoreConfig(self.configFile)
94 config = DatastoreConfig(self.configFile, mergeDefaults=False)
95 newroot = "/random/location"
96 self.datastoreType.setConfigRoot(newroot, config, full)
97 if self.rootKeys:
98 for k in self.rootKeys:
99 self.assertIn(newroot, config[k])
101 def testConstructor(self):
102 datastore = self.makeDatastore()
103 self.assertIsNotNone(datastore)
104 self.assertIs(datastore.isEphemeral, self.isEphemeral)
106 def testConfigurationValidation(self):
107 datastore = self.makeDatastore()
108 sc = self.storageClassFactory.getStorageClass("ThingOne")
109 datastore.validateConfiguration([sc])
111 sc2 = self.storageClassFactory.getStorageClass("ThingTwo")
112 if self.validationCanFail:
113 with self.assertRaises(DatastoreValidationError):
114 datastore.validateConfiguration([sc2], logFailures=True)
116 dimensions = self.universe.extract(("visit", "physical_filter"))
117 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
118 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
119 datastore.validateConfiguration([ref])
121 def testParameterValidation(self):
122 """Check that parameters are validated"""
123 sc = self.storageClassFactory.getStorageClass("ThingOne")
124 dimensions = self.universe.extract(("visit", "physical_filter"))
125 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
126 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
127 datastore = self.makeDatastore()
128 data = {1: 2, 3: 4}
129 datastore.put(data, ref)
130 newdata = datastore.get(ref)
131 self.assertEqual(data, newdata)
132 with self.assertRaises(KeyError):
133 newdata = datastore.get(ref, parameters={"missing": 5})
135 def testBasicPutGet(self):
136 metrics = makeExampleMetrics()
137 datastore = self.makeDatastore()
139 # Create multiple storage classes for testing different formulations
140 storageClasses = [self.storageClassFactory.getStorageClass(sc)
141 for sc in ("StructuredData",
142 "StructuredDataJson",
143 "StructuredDataPickle")]
145 dimensions = self.universe.extract(("visit", "physical_filter"))
146 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
148 for sc in storageClasses:
149 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
150 print("Using storageClass: {}".format(sc.name))
151 datastore.put(metrics, ref)
153 # Does it exist?
154 self.assertTrue(datastore.exists(ref))
156 # Get
157 metricsOut = datastore.get(ref, parameters=None)
158 self.assertEqual(metrics, metricsOut)
160 uri = datastore.getURI(ref)
161 self.assertEqual(uri.scheme, self.uriScheme)
163 # Get a component -- we need to construct new refs for them
164 # with derived storage classes but with parent ID
165 for comp in ("data", "output"):
166 compRef = ref.makeComponentRef(comp)
167 output = datastore.get(compRef)
168 self.assertEqual(output, getattr(metricsOut, comp))
170 uri = datastore.getURI(compRef)
171 self.assertEqual(uri.scheme, self.uriScheme)
173 storageClass = sc
175 # Check that we can put a metric with None in a component and
176 # get it back as None
177 metricsNone = makeExampleMetrics(use_none=True)
178 dataIdNone = {"instrument": "dummy", "visit": 54, "physical_filter": "V"}
179 refNone = self.makeDatasetRef("metric", dimensions, sc, dataIdNone, conform=False)
180 datastore.put(metricsNone, refNone)
182 comp = "data"
183 for comp in ("data", "output"):
184 compRef = refNone.makeComponentRef(comp)
185 output = datastore.get(compRef)
186 self.assertEqual(output, getattr(metricsNone, comp))
188 # Check that a put fails if the dataset type is not supported
189 if self.hasUnsupportedPut:
190 sc = StorageClass("UnsupportedSC", pytype=type(metrics))
191 ref = self.makeDatasetRef("unsupportedType", dimensions, sc, dataId)
192 with self.assertRaises(DatasetTypeNotSupportedError):
193 datastore.put(metrics, ref)
195 # These should raise
196 ref = self.makeDatasetRef("metrics", dimensions, storageClass, dataId, id=10000)
197 with self.assertRaises(FileNotFoundError):
198 # non-existing file
199 datastore.get(ref)
201 # Get a URI from it
202 uri = datastore.getURI(ref, predict=True)
203 self.assertEqual(uri.scheme, self.uriScheme)
205 with self.assertRaises(FileNotFoundError):
206 datastore.getURI(ref)
208 def testDisassembly(self):
209 """Test disassembly within datastore."""
210 metrics = makeExampleMetrics()
211 if self.isEphemeral:
212 # in-memory datastore does not disassemble
213 return
215 # Create multiple storage classes for testing different formulations
216 # of composites. One of these will not disassemble to provide
217 # a reference.
218 storageClasses = [self.storageClassFactory.getStorageClass(sc)
219 for sc in ("StructuredComposite",
220 "StructuredCompositeTestA",
221 "StructuredCompositeTestB",
222 "StructuredCompositeReadComp",
223 "StructuredData", # No disassembly
224 "StructuredCompositeReadCompNoDisassembly",
225 )]
227 # Create the test datastore
228 datastore = self.makeDatastore()
230 # Dummy dataId
231 dimensions = self.universe.extract(("visit", "physical_filter"))
232 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"}
234 for i, sc in enumerate(storageClasses):
235 with self.subTest(storageClass=sc.name):
236 # Create a different dataset type each time round
237 # so that a test failure in this subtest does not trigger
238 # a cascade of tests because of file clashes
239 ref = self.makeDatasetRef(f"metric_comp_{i}", dimensions, sc, dataId,
240 conform=False)
242 disassembled = sc.name not in {"StructuredData", "StructuredCompositeReadCompNoDisassembly"}
244 datastore.put(metrics, ref)
246 baseURI, compURIs = datastore.getURIs(ref)
247 if disassembled:
248 self.assertIsNone(baseURI)
249 self.assertEqual(set(compURIs), {"data", "output", "summary"})
250 else:
251 self.assertIsNotNone(baseURI)
252 self.assertEqual(compURIs, {})
254 metrics_get = datastore.get(ref)
255 self.assertEqual(metrics_get, metrics)
257 # Retrieve the composite with read parameter
258 stop = 4
259 metrics_get = datastore.get(ref, parameters={"slice": slice(stop)})
260 self.assertEqual(metrics_get.summary, metrics.summary)
261 self.assertEqual(metrics_get.output, metrics.output)
262 self.assertEqual(metrics_get.data, metrics.data[:stop])
264 # Retrieve a component
265 data = datastore.get(ref.makeComponentRef("data"))
266 self.assertEqual(data, metrics.data)
268 # On supported storage classes attempt to access a read
269 # only component
270 if "ReadComp" in sc.name:
271 cRef = ref.makeComponentRef("counter")
272 counter = datastore.get(cRef)
273 self.assertEqual(counter, len(metrics.data))
275 counter = datastore.get(cRef, parameters={"slice": slice(stop)})
276 self.assertEqual(counter, stop)
278 datastore.remove(ref)
280 def testRegistryCompositePutGet(self):
281 """Tests the case where registry disassembles and puts to datastore.
282 """
283 metrics = makeExampleMetrics()
284 datastore = self.makeDatastore()
286 # Create multiple storage classes for testing different formulations
287 # of composites
288 storageClasses = [self.storageClassFactory.getStorageClass(sc)
289 for sc in ("StructuredComposite",
290 "StructuredCompositeTestA",
291 "StructuredCompositeTestB",
292 )]
294 dimensions = self.universe.extract(("visit", "physical_filter"))
295 dataId = {"instrument": "dummy", "visit": 428, "physical_filter": "R"}
297 for sc in storageClasses:
298 print("Using storageClass: {}".format(sc.name))
299 ref = self.makeDatasetRef("metric", dimensions, sc, dataId,
300 conform=False)
302 components = sc.assembler().disassemble(metrics)
303 self.assertTrue(components)
305 compsRead = {}
306 for compName, compInfo in components.items():
307 compRef = self.makeDatasetRef(ref.datasetType.componentTypeName(compName), dimensions,
308 components[compName].storageClass, dataId,
309 conform=False)
311 print("Writing component {} with {}".format(compName, compRef.datasetType.storageClass.name))
312 datastore.put(compInfo.component, compRef)
314 uri = datastore.getURI(compRef)
315 self.assertEqual(uri.scheme, self.uriScheme)
317 compsRead[compName] = datastore.get(compRef)
319 # We can generate identical files for each storage class
320 # so remove the component here
321 datastore.remove(compRef)
323 # combine all the components we read back into a new composite
324 metricsOut = sc.assembler().assemble(compsRead)
325 self.assertEqual(metrics, metricsOut)
327 def testRemove(self):
328 metrics = makeExampleMetrics()
329 datastore = self.makeDatastore()
330 # Put
331 dimensions = self.universe.extract(("visit", "physical_filter"))
332 dataId = {"instrument": "dummy", "visit": 638, "physical_filter": "U"}
334 sc = self.storageClassFactory.getStorageClass("StructuredData")
335 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
336 datastore.put(metrics, ref)
338 # Does it exist?
339 self.assertTrue(datastore.exists(ref))
341 # Get
342 metricsOut = datastore.get(ref)
343 self.assertEqual(metrics, metricsOut)
344 # Remove
345 datastore.remove(ref)
347 # Does it exist?
348 self.assertFalse(datastore.exists(ref))
350 # Do we now get a predicted URI?
351 uri = datastore.getURI(ref, predict=True)
352 self.assertEqual(uri.fragment, "predicted")
354 # Get should now fail
355 with self.assertRaises(FileNotFoundError):
356 datastore.get(ref)
357 # Can only delete once
358 with self.assertRaises(FileNotFoundError):
359 datastore.remove(ref)
361 def testTransfer(self):
362 metrics = makeExampleMetrics()
364 dimensions = self.universe.extract(("visit", "physical_filter"))
365 dataId = {"instrument": "dummy", "visit": 2048, "physical_filter": "Uprime"}
367 sc = self.storageClassFactory.getStorageClass("StructuredData")
368 ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
370 inputDatastore = self.makeDatastore("test_input_datastore")
371 outputDatastore = self.makeDatastore("test_output_datastore")
373 inputDatastore.put(metrics, ref)
374 outputDatastore.transfer(inputDatastore, ref)
376 metricsOut = outputDatastore.get(ref)
377 self.assertEqual(metrics, metricsOut)
379 def testBasicTransaction(self):
380 datastore = self.makeDatastore()
381 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
382 dimensions = self.universe.extract(("visit", "physical_filter"))
383 nDatasets = 6
384 dataIds = [{"instrument": "dummy", "visit": i, "physical_filter": "V"} for i in range(nDatasets)]
385 data = [(self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False),
386 makeExampleMetrics(),)
387 for dataId in dataIds]
388 succeed = data[:nDatasets//2]
389 fail = data[nDatasets//2:]
390 # All datasets added in this transaction should continue to exist
391 with datastore.transaction():
392 for ref, metrics in succeed:
393 datastore.put(metrics, ref)
394 # Whereas datasets added in this transaction should not
395 with self.assertRaises(TransactionTestError):
396 with datastore.transaction():
397 for ref, metrics in fail:
398 datastore.put(metrics, ref)
399 raise TransactionTestError("This should propagate out of the context manager")
400 # Check for datasets that should exist
401 for ref, metrics in succeed:
402 # Does it exist?
403 self.assertTrue(datastore.exists(ref))
404 # Get
405 metricsOut = datastore.get(ref, parameters=None)
406 self.assertEqual(metrics, metricsOut)
407 # URI
408 uri = datastore.getURI(ref)
409 self.assertEqual(uri.scheme, self.uriScheme)
410 # Check for datasets that should not exist
411 for ref, _ in fail:
412 # These should raise
413 with self.assertRaises(FileNotFoundError):
414 # non-existing file
415 datastore.get(ref)
416 with self.assertRaises(FileNotFoundError):
417 datastore.getURI(ref)
419 def testNestedTransaction(self):
420 datastore = self.makeDatastore()
421 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
422 dimensions = self.universe.extract(("visit", "physical_filter"))
423 metrics = makeExampleMetrics()
425 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
426 refBefore = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
427 conform=False)
428 datastore.put(metrics, refBefore)
429 with self.assertRaises(TransactionTestError):
430 with datastore.transaction():
431 dataId = {"instrument": "dummy", "visit": 1, "physical_filter": "V"}
432 refOuter = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
433 conform=False)
434 datastore.put(metrics, refOuter)
435 with datastore.transaction():
436 dataId = {"instrument": "dummy", "visit": 2, "physical_filter": "V"}
437 refInner = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
438 conform=False)
439 datastore.put(metrics, refInner)
440 # All datasets should exist
441 for ref in (refBefore, refOuter, refInner):
442 metricsOut = datastore.get(ref, parameters=None)
443 self.assertEqual(metrics, metricsOut)
444 raise TransactionTestError("This should roll back the transaction")
445 # Dataset(s) inserted before the transaction should still exist
446 metricsOut = datastore.get(refBefore, parameters=None)
447 self.assertEqual(metrics, metricsOut)
448 # But all datasets inserted during the (rolled back) transaction
449 # should be gone
450 with self.assertRaises(FileNotFoundError):
451 datastore.get(refOuter)
452 with self.assertRaises(FileNotFoundError):
453 datastore.get(refInner)
455 def _prepareIngestTest(self):
456 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
457 dimensions = self.universe.extract(("visit", "physical_filter"))
458 metrics = makeExampleMetrics()
459 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
460 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
461 return metrics, ref
463 def runIngestTest(self, func, expectOutput=True):
464 metrics, ref = self._prepareIngestTest()
465 with lsst.utils.tests.getTempFilePath(".yaml", expectOutput=expectOutput) as path:
466 with open(path, 'w') as fd:
467 yaml.dump(metrics._asdict(), stream=fd)
468 func(metrics, path, ref)
470 def testIngestNoTransfer(self):
471 """Test ingesting existing files with no transfer.
472 """
473 for mode in (None, "auto"):
475 # Some datastores have auto but can't do in place transfer
476 if mode == "auto" and "auto" in self.ingestTransferModes and not self.canIngestNoTransferAuto:
477 continue
479 with self.subTest(mode=mode):
480 datastore = self.makeDatastore()
482 def succeed(obj, path, ref):
483 """Ingest a file already in the datastore root."""
484 # first move it into the root, and adjust the path
485 # accordingly
486 path = shutil.copy(path, datastore.root)
487 path = os.path.relpath(path, start=datastore.root)
488 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
489 self.assertEqual(obj, datastore.get(ref))
491 def failInputDoesNotExist(obj, path, ref):
492 """Can't ingest files if we're given a bad path."""
493 with self.assertRaises(FileNotFoundError):
494 datastore.ingest(FileDataset(path="this-file-does-not-exist.yaml", refs=ref),
495 transfer=mode)
496 self.assertFalse(datastore.exists(ref))
498 def failOutsideRoot(obj, path, ref):
499 """Can't ingest files outside of datastore root unless
500 auto."""
501 if mode == "auto":
502 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
503 self.assertTrue(datastore.exists(ref))
504 else:
505 with self.assertRaises(RuntimeError):
506 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
507 self.assertFalse(datastore.exists(ref))
509 def failNotImplemented(obj, path, ref):
510 with self.assertRaises(NotImplementedError):
511 datastore.ingest(FileDataset(path=path, refs=ref), transfer=mode)
513 if mode in self.ingestTransferModes:
514 self.runIngestTest(failOutsideRoot)
515 self.runIngestTest(failInputDoesNotExist)
516 self.runIngestTest(succeed)
517 else:
518 self.runIngestTest(failNotImplemented)
520 def testIngestTransfer(self):
521 """Test ingesting existing files after transferring them.
522 """
523 for mode in ("copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto"):
524 with self.subTest(mode=mode):
525 datastore = self.makeDatastore(mode)
527 def succeed(obj, path, ref):
528 """Ingest a file by transferring it to the template
529 location."""
530 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
531 self.assertEqual(obj, datastore.get(ref))
533 def failInputDoesNotExist(obj, path, ref):
534 """Can't ingest files if we're given a bad path."""
535 with self.assertRaises(FileNotFoundError):
536 # Ensure the file does not look like it is in
537 # datastore for auto mode
538 datastore.ingest(FileDataset(path="../this-file-does-not-exist.yaml", refs=ref),
539 transfer=mode)
540 self.assertFalse(datastore.exists(ref))
542 def failOutputExists(obj, path, ref):
543 """Can't ingest files if transfer destination already
544 exists."""
545 with self.assertRaises(FileExistsError):
546 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
547 self.assertFalse(datastore.exists(ref))
549 def failNotImplemented(obj, path, ref):
550 with self.assertRaises(NotImplementedError):
551 datastore.ingest(FileDataset(path=os.path.abspath(path), refs=ref), transfer=mode)
553 if mode in self.ingestTransferModes:
554 self.runIngestTest(failInputDoesNotExist)
555 self.runIngestTest(succeed, expectOutput=(mode != "move"))
556 self.runIngestTest(failOutputExists)
557 else:
558 self.runIngestTest(failNotImplemented)
560 def testIngestSymlinkOfSymlink(self):
561 """Special test for symlink to a symlink ingest"""
562 metrics, ref = self._prepareIngestTest()
563 # The aim of this test is to create a dataset on disk, then
564 # create a symlink to it and finally ingest the symlink such that
565 # the symlink in the datastore points to the original dataset.
566 for mode in ("symlink", "relsymlink"):
567 if mode not in self.ingestTransferModes:
568 continue
570 print(f"Trying mode {mode}")
571 with lsst.utils.tests.getTempFilePath(".yaml") as realpath:
572 with open(realpath, 'w') as fd:
573 yaml.dump(metrics._asdict(), stream=fd)
574 with lsst.utils.tests.getTempFilePath(".yaml") as sympath:
575 os.symlink(os.path.abspath(realpath), sympath)
577 datastore = self.makeDatastore()
578 datastore.ingest(FileDataset(path=os.path.abspath(sympath), refs=ref), transfer=mode)
580 uri = datastore.getURI(ref)
581 self.assertTrue(not uri.scheme or uri.scheme == "file", f"Check {uri.scheme}")
582 self.assertTrue(os.path.islink(uri.path))
584 linkTarget = os.readlink(uri.path)
585 if mode == "relsymlink":
586 self.assertFalse(os.path.isabs(linkTarget))
587 else:
588 self.assertEqual(linkTarget, os.path.abspath(realpath))
590 # Check that we can get the dataset back regardless of mode
591 metric2 = datastore.get(ref)
592 self.assertEqual(metric2, metrics)
594 # Cleanup the file for next time round loop
595 # since it will get the same file name in store
596 datastore.remove(ref)
599class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase):
600 """PosixDatastore specialization"""
601 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
602 uriScheme = "file"
603 canIngestNoTransferAuto = True
604 ingestTransferModes = (None, "copy", "move", "link", "hardlink", "symlink", "relsymlink", "auto")
605 isEphemeral = False
606 rootKeys = ("root",)
607 validationCanFail = True
609 def setUp(self):
610 # Override the working directory before calling the base class
611 self.root = tempfile.mkdtemp(dir=TESTDIR)
612 super().setUp()
615class PosixDatastoreNoChecksumsTestCase(PosixDatastoreTestCase):
616 """Posix datastore tests but with checksums disabled."""
617 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreNoChecksums.yaml")
619 def testChecksum(self):
620 """Ensure that checksums have not been calculated."""
622 datastore = self.makeDatastore()
623 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
624 dimensions = self.universe.extract(("visit", "physical_filter"))
625 metrics = makeExampleMetrics()
627 dataId = {"instrument": "dummy", "visit": 0, "physical_filter": "V"}
628 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId,
629 conform=False)
631 # Configuration should have disabled checksum calculation
632 datastore.put(metrics, ref)
633 infos = datastore.getStoredItemsInfo(ref)
634 self.assertIsNone(infos[0].checksum)
636 # Remove put back but with checksums enabled explicitly
637 datastore.remove(ref)
638 datastore.useChecksum = True
639 datastore.put(metrics, ref)
641 infos = datastore.getStoredItemsInfo(ref)
642 self.assertIsNotNone(infos[0].checksum)
645class CleanupPosixDatastoreTestCase(DatastoreTestsBase, unittest.TestCase):
646 configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")
648 def setUp(self):
649 # Override the working directory before calling the base class
650 self.root = tempfile.mkdtemp(dir=TESTDIR)
651 super().setUp()
653 def testCleanup(self):
654 """Test that a failed formatter write does cleanup a partial file."""
655 metrics = makeExampleMetrics()
656 datastore = self.makeDatastore()
658 storageClass = self.storageClassFactory.getStorageClass("StructuredData")
660 dimensions = self.universe.extract(("visit", "physical_filter"))
661 dataId = {"instrument": "dummy", "visit": 52, "physical_filter": "V"}
663 ref = self.makeDatasetRef("metric", dimensions, storageClass, dataId, conform=False)
665 # Determine where the file will end up (we assume Formatters use
666 # the same file extension)
667 expectedUri = datastore.getURI(ref, predict=True)
668 self.assertEqual(expectedUri.fragment, "predicted")
670 expectedFile = expectedUri.path
671 self.assertTrue(expectedFile.endswith(".yaml"),
672 f"Is there a file extension in {expectedUri}")
674 # Try formatter that fails and formatter that fails and leaves
675 # a file behind
676 for formatter in (BadWriteFormatter, BadNoWriteFormatter):
677 with self.subTest(formatter=formatter):
679 # Monkey patch the formatter
680 datastore.formatterFactory.registerFormatter(ref.datasetType, formatter,
681 overwrite=True)
683 # Try to put the dataset, it should fail
684 with self.assertRaises(Exception):
685 datastore.put(metrics, ref)
687 # Check that there is no file on disk
688 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
690 # Check that there is a directory
691 self.assertTrue(os.path.exists(os.path.dirname(expectedFile)),
692 f"Check for existence of directory {os.path.dirname(expectedFile)}")
694 # Force YamlFormatter and check that this time a file is written
695 datastore.formatterFactory.registerFormatter(ref.datasetType, YamlFormatter,
696 overwrite=True)
697 datastore.put(metrics, ref)
698 self.assertTrue(os.path.exists(expectedFile), f"Check for existence of {expectedFile}")
699 datastore.remove(ref)
700 self.assertFalse(os.path.exists(expectedFile), f"Check for existence of now removed {expectedFile}")
703class InMemoryDatastoreTestCase(DatastoreTests, unittest.TestCase):
704 """PosixDatastore specialization"""
705 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastore.yaml")
706 uriScheme = "mem"
707 hasUnsupportedPut = False
708 ingestTransferModes = ()
709 isEphemeral = True
710 rootKeys = None
711 validationCanFail = False
714class ChainedDatastoreTestCase(PosixDatastoreTestCase):
715 """ChainedDatastore specialization using a POSIXDatastore"""
716 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore.yaml")
717 hasUnsupportedPut = False
718 canIngestNoTransferAuto = False
719 ingestTransferModes = ("copy", "hardlink", "symlink", "relsymlink", "link", "auto")
720 isEphemeral = False
721 rootKeys = (".datastores.1.root", ".datastores.2.root")
722 validationCanFail = True
725class ChainedDatastoreMemoryTestCase(InMemoryDatastoreTestCase):
726 """ChainedDatastore specialization using all InMemoryDatastore"""
727 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2.yaml")
728 validationCanFail = False
731class DatastoreConstraintsTests(DatastoreTestsBase):
732 """Basic tests of constraints model of Datastores."""
734 def testConstraints(self):
735 """Test constraints model. Assumes that each test class has the
736 same constraints."""
737 metrics = makeExampleMetrics()
738 datastore = self.makeDatastore()
740 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
741 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
742 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
743 dataId = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
745 # Write empty file suitable for ingest check (JSON and YAML variants)
746 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
747 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
748 for datasetTypeName, sc, accepted in (("metric", sc1, True), ("metric2", sc1, False),
749 ("metric33", sc1, True), ("metric2", sc2, True)):
750 # Choose different temp file depending on StorageClass
751 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
753 with self.subTest(datasetTypeName=datasetTypeName, storageClass=sc.name, file=testfile.name):
754 ref = self.makeDatasetRef(datasetTypeName, dimensions, sc, dataId, conform=False)
755 if accepted:
756 datastore.put(metrics, ref)
757 self.assertTrue(datastore.exists(ref))
758 datastore.remove(ref)
760 # Try ingest
761 if self.canIngest:
762 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
763 self.assertTrue(datastore.exists(ref))
764 datastore.remove(ref)
765 else:
766 with self.assertRaises(DatasetTypeNotSupportedError):
767 datastore.put(metrics, ref)
768 self.assertFalse(datastore.exists(ref))
770 # Again with ingest
771 if self.canIngest:
772 with self.assertRaises(DatasetTypeNotSupportedError):
773 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
774 self.assertFalse(datastore.exists(ref))
777class PosixDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
778 """PosixDatastore specialization"""
779 configFile = os.path.join(TESTDIR, "config/basic/posixDatastoreP.yaml")
780 canIngest = True
782 def setUp(self):
783 # Override the working directory before calling the base class
784 self.root = tempfile.mkdtemp(dir=TESTDIR)
785 super().setUp()
788class InMemoryDatastoreConstraintsTestCase(DatastoreConstraintsTests, unittest.TestCase):
789 """InMemoryDatastore specialization"""
790 configFile = os.path.join(TESTDIR, "config/basic/inMemoryDatastoreP.yaml")
791 canIngest = False
794class ChainedDatastoreConstraintsNativeTestCase(PosixDatastoreConstraintsTestCase):
795 """ChainedDatastore specialization using a POSIXDatastore and constraints
796 at the ChainedDatstore """
797 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePa.yaml")
800class ChainedDatastoreConstraintsTestCase(PosixDatastoreConstraintsTestCase):
801 """ChainedDatastore specialization using a POSIXDatastore"""
802 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastoreP.yaml")
805class ChainedDatastoreMemoryConstraintsTestCase(InMemoryDatastoreConstraintsTestCase):
806 """ChainedDatastore specialization using all InMemoryDatastore"""
807 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastore2P.yaml")
808 canIngest = False
811class ChainedDatastorePerStoreConstraintsTests(DatastoreTestsBase, unittest.TestCase):
812 """Test that a chained datastore can control constraints per-datastore
813 even if child datastore would accept."""
815 configFile = os.path.join(TESTDIR, "config/basic/chainedDatastorePb.yaml")
817 def setUp(self):
818 # Override the working directory before calling the base class
819 self.root = tempfile.mkdtemp(dir=TESTDIR)
820 super().setUp()
822 def testConstraints(self):
823 """Test chained datastore constraints model."""
824 metrics = makeExampleMetrics()
825 datastore = self.makeDatastore()
827 sc1 = self.storageClassFactory.getStorageClass("StructuredData")
828 sc2 = self.storageClassFactory.getStorageClass("StructuredDataJson")
829 dimensions = self.universe.extract(("visit", "physical_filter", "instrument"))
830 dataId1 = {"visit": 52, "physical_filter": "V", "instrument": "DummyCamComp"}
831 dataId2 = {"visit": 52, "physical_filter": "V", "instrument": "HSC"}
833 # Write empty file suitable for ingest check (JSON and YAML variants)
834 testfile_y = tempfile.NamedTemporaryFile(suffix=".yaml")
835 testfile_j = tempfile.NamedTemporaryFile(suffix=".json")
837 for typeName, dataId, sc, accept, ingest in (("metric", dataId1, sc1, (False, True, False), True),
838 ("metric2", dataId1, sc1, (False, False, False), False),
839 ("metric2", dataId2, sc1, (True, False, False), False),
840 ("metric33", dataId2, sc2, (True, True, False), True),
841 ("metric2", dataId1, sc2, (False, True, False), True)):
843 # Choose different temp file depending on StorageClass
844 testfile = testfile_j if sc.name.endswith("Json") else testfile_y
846 with self.subTest(datasetTypeName=typeName, dataId=dataId, sc=sc.name):
847 ref = self.makeDatasetRef(typeName, dimensions, sc, dataId,
848 conform=False)
849 if any(accept):
850 datastore.put(metrics, ref)
851 self.assertTrue(datastore.exists(ref))
853 # Check each datastore inside the chained datastore
854 for childDatastore, expected in zip(datastore.datastores, accept):
855 self.assertEqual(childDatastore.exists(ref), expected,
856 f"Testing presence of {ref} in datastore {childDatastore.name}")
858 datastore.remove(ref)
860 # Check that ingest works
861 if ingest:
862 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
863 self.assertTrue(datastore.exists(ref))
865 # Check each datastore inside the chained datastore
866 for childDatastore, expected in zip(datastore.datastores, accept):
867 # Ephemeral datastores means InMemory at the moment
868 # and that does not accept ingest of files.
869 if childDatastore.isEphemeral:
870 expected = False
871 self.assertEqual(childDatastore.exists(ref), expected,
872 f"Testing presence of ingested {ref} in datastore"
873 f" {childDatastore.name}")
875 datastore.remove(ref)
876 else:
877 with self.assertRaises(DatasetTypeNotSupportedError):
878 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
880 else:
881 with self.assertRaises(DatasetTypeNotSupportedError):
882 datastore.put(metrics, ref)
883 self.assertFalse(datastore.exists(ref))
885 # Again with ingest
886 with self.assertRaises(DatasetTypeNotSupportedError):
887 datastore.ingest(FileDataset(testfile.name, [ref]), transfer="link")
888 self.assertFalse(datastore.exists(ref))
891if __name__ == "__main__": 891 ↛ 892line 891 didn't jump to line 892, because the condition on line 891 was never true
892 unittest.main()