Coverage for tests/test_butler.py : 27%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
# This file is part of daf_butler. # # Developed for the LSST Data Management System. # This product includes software developed by the LSST Project # (http://www.lsst.org). # See the COPYRIGHT file at the top-level directory of this distribution # for details of code ownership. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>.
"""
return MetricsExample({"AM1": 5.2, "AM2": 30.6}, {"a": [1, 2, 3], "b": {"blue": 5, "red": "green"}}, [563, 234, 456.7, 752, 8, 9, 27] )
"""Specific error for testing transactions, to prevent misdiagnosing that might otherwise occur when a standard exception is used. """
"""Tests for Butler. """
def addDatasetType(datasetTypeName, dataUnits, storageClass, registry): """Create a DatasetType and register it """ datasetType = DatasetType(datasetTypeName, dataUnits, storageClass) registry.registerDatasetType(datasetType) return datasetType
def setUpClass(cls): cls.storageClassFactory = StorageClassFactory() cls.storageClassFactory.addFromConfig(cls.configFile)
datasetTypeName = datasetRef.datasetType.name dataId = datasetRef.dataId for component in components: compTypeName = DatasetType.nameWithComponent(datasetTypeName, component) result = butler.get(compTypeName, dataId) self.assertEqual(result, getattr(reference, component))
"""Create a new butler root for each test.""" if self.useTempRoot: self.root = tempfile.mkdtemp(dir=TESTDIR) Butler.makeRepo(self.root, config=Config(self.configFile)) self.tmpConfigFile = os.path.join(self.root, "butler.yaml") else: self.root = None self.tmpConfigFile = self.configFile
if self.root is not None and os.path.exists(self.root): shutil.rmtree(self.root, ignore_errors=True)
"""Independent test of constructor. """ butler = Butler(self.tmpConfigFile) self.assertIsInstance(butler, Butler)
storageClass = self.storageClassFactory.getStorageClass("StructuredDataNoComponents") self.runPutGetTest(storageClass, "test_metric")
storageClass = self.storageClassFactory.getStorageClass("StructuredData") self.runPutGetTest(storageClass, "test_metric")
storageClass = self.storageClassFactory.getStorageClass("StructuredComposite") self.runPutGetTest(storageClass, "test_metric_comp")
butler = Butler(self.tmpConfigFile) # Create and register a DatasetType dataUnits = ("Camera", "Visit")
# We can not delete datasets so for now create two so we can do # two puts. self.addDatasetType(datasetTypeName, dataUnits, storageClass, butler.registry)
datasetTypeName2 = datasetTypeName + "2" self.addDatasetType(datasetTypeName2, dataUnits, storageClass, butler.registry)
# Add needed DataUnits butler.registry.addDataUnitEntry("Camera", {"camera": "DummyCamComp"}) butler.registry.addDataUnitEntry("PhysicalFilter", {"camera": "DummyCamComp", "physical_filter": "d-r"}) butler.registry.addDataUnitEntry("Visit", {"camera": "DummyCamComp", "visit": 423, "physical_filter": "d-r"})
# Create and store a dataset metric = makeExampleMetrics() dataId = {"camera": "DummyCamComp", "visit": 423}
# Create a DatasetRef for put datasetType2 = butler.registry.getDatasetType(datasetTypeName2) ref2 = DatasetRef(datasetType2, dataId, id=None)
# Put with a preexisting id should fail with self.assertRaises(ValueError): butler.put(metric, DatasetRef(datasetType2, dataId, id=100))
# Put the dataset once as a DatasetRef and once as a dataId for args in ((ref2,), (datasetTypeName, dataId)): ref = butler.put(metric, *args) self.assertIsInstance(ref, DatasetRef)
# Test getDirect metricOut = butler.getDirect(ref) self.assertEqual(metric, metricOut) # Test get metricOut = butler.get(ref.datasetType.name, dataId) self.assertEqual(metric, metricOut) # Test get with a datasetRef metricOut = butler.get(ref) self.assertEqual(metric, metricOut)
# Check we can get components if storageClass.isComposite(): self.assertGetComponents(butler, ref, ("summary", "data", "output"), metric)
# Get with parameters stop = 4 sliced = butler.get(ref2, parameters={"slice": slice(stop)}) self.assertNotEqual(metric, sliced) self.assertEqual(metric.summary, sliced.summary) self.assertEqual(metric.output, sliced.output) self.assertEqual(metric.data[:stop], sliced.data)
# Combining a DatasetRef with a dataId should fail with self.assertRaises(ValueError): butler.get(ref, dataId) # Getting with an explicit ref should fail if the id doesn't match with self.assertRaises(ValueError): butler.get(DatasetRef(ref.datasetType, ref.dataId, id=101))
# Getting a dataset with unknown parameters should fail with self.assertRaises(KeyError): butler.get(ref, parameters={"unsupported": True})
"""Test pickle support. """ butler = Butler(self.tmpConfigFile) butlerOut = pickle.loads(pickle.dumps(butler)) self.assertIsInstance(butlerOut, Butler) self.assertEqual(butlerOut.config, butler.config)
butler = Butler(self.tmpConfigFile) datasetTypeName = "test_metric" dataUnits = ("Camera", "Visit") dataUnitEntries = (("Camera", {"camera": "DummyCam"}), ("PhysicalFilter", {"camera": "DummyCam", "physical_filter": "d-r"}), ("Visit", {"camera": "DummyCam", "visit": 42, "physical_filter": "d-r"})) storageClass = self.storageClassFactory.getStorageClass("StructuredData") metric = makeExampleMetrics() dataId = {"camera": "DummyCam", "visit": 42} with self.assertRaises(TransactionTestError): with butler.transaction(): # Create and register a DatasetType datasetType = self.addDatasetType(datasetTypeName, dataUnits, storageClass, butler.registry) # Add needed DataUnits for name, value in dataUnitEntries: butler.registry.addDataUnitEntry(name, value) # Store a dataset ref = butler.put(metric, datasetTypeName, dataId) self.assertIsInstance(ref, DatasetRef) # Test getDirect metricOut = butler.getDirect(ref) self.assertEqual(metric, metricOut) # Test get metricOut = butler.get(datasetTypeName, dataId) self.assertEqual(metric, metricOut) # Check we can get components self.assertGetComponents(butler, ref, ("summary", "data", "output"), metric) raise TransactionTestError("This should roll back the entire transaction")
with self.assertRaises(KeyError): butler.registry.getDatasetType(datasetTypeName) for name, value in dataUnitEntries: self.assertIsNone(butler.registry.findDataUnitEntry(name, value)) # Should raise KeyError for missing DatasetType with self.assertRaises(KeyError): butler.get(datasetTypeName, dataId) # Also check explicitly if Dataset entry is missing self.assertIsNone(butler.registry.find(butler.collection, datasetType, dataId)) # Direct retrieval should not find the file in the Datastore with self.assertRaises(FileNotFoundError): butler.getDirect(ref)
"""Test that we can write butler configuration to a new repository via the Butler.makeRepo interface and then instantiate a butler from the repo root. """ # Do not run the test if we know this datastore configuration does # not support a file system root if self.fullConfigKey is None: return
# Remove the file created in setUp os.unlink(self.tmpConfigFile)
Butler.makeRepo(self.root, config=Config(self.configFile)) limited = Config(self.configFile) butler1 = Butler(self.root, collection="ingest") Butler.makeRepo(self.root, standalone=True, createRegistry=False, config=Config(self.configFile)) full = Config(self.tmpConfigFile) butler2 = Butler(self.root, collection="ingest") # Butlers should have the same configuration regardless of whether # defaults were expanded. self.assertEqual(butler1.config, butler2.config) # Config files loaded directly should not be the same. self.assertNotEqual(limited, full) # Make sure "limited" doesn't have a few keys we know it should be # inheriting from defaults. self.assertIn(self.fullConfigKey, full) self.assertNotIn(self.fullConfigKey, limited)
butler = Butler(self.configFile) if self.datastoreStr is not None: self.assertIn(self.datastoreStr, str(butler)) if self.registryStr is not None: self.assertIn(self.registryStr, str(butler))
"""PosixDatastore specialization of a butler"""
"""InMemoryDatastore specialization of a butler"""
"""PosixDatastore specialization"""
lsst.utils.tests.init()
lsst.utils.tests.init() unittest.main() |