Coverage for tests/test_storageClass.py: 12%
199 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 02:30 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-03 02:30 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24import pickle
25import unittest
27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory
28from lsst.daf.butler.tests import MetricsExample
29from lsst.utils.introspection import get_full_type_name
31"""Tests related to the StorageClass infrastructure.
32"""
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class PythonType:
38 """A dummy class to test the registry of Python types."""
40 pass
43class StorageClassFactoryTestCase(unittest.TestCase):
44 """Tests of the storage class infrastructure."""
46 def testCreation(self):
47 """Test that we can dynamically create storage class subclasses.
49 This is critical for testing the factory functions."""
50 className = "TestImage"
51 sc = StorageClass(className, pytype=dict)
52 self.assertIsInstance(sc, StorageClass)
53 self.assertEqual(sc.name, className)
54 self.assertEqual(str(sc), className)
55 self.assertFalse(sc.components)
56 self.assertTrue(sc.validateInstance({}))
57 self.assertFalse(sc.validateInstance(""))
59 r = repr(sc)
60 self.assertIn("StorageClass", r)
61 self.assertIn(className, r)
62 self.assertNotIn("parameters", r)
63 self.assertIn("pytype='dict'", r)
65 # Ensure we do not have a delegate
66 with self.assertRaises(TypeError):
67 sc.delegate()
69 # Allow no definition of python type
70 scn = StorageClass(className)
71 self.assertIs(scn.pytype, object)
73 # Include some components
74 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc})
75 self.assertIn("comp1", scc.components)
76 r = repr(scc)
77 self.assertIn("comp1", r)
78 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r)
80 # Ensure that we have a delegate
81 self.assertIsInstance(scc.delegate(), StorageClassDelegate)
83 # Check we can create a storageClass using the name of an importable
84 # type.
85 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
86 self.assertIsInstance(sc2.pytype(), StorageClassFactory)
87 self.assertIn("butler.core", repr(sc2))
89 def testParameters(self):
90 """Test that we can set parameters and validate them"""
91 pt = ("a", "b")
92 ps = {"a", "b"}
93 pl = ["a", "b"]
94 for p in (pt, ps, pl):
95 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p)
96 self.assertEqual(sc1.parameters, ps)
97 sc1.validateParameters(p)
99 sc1.validateParameters()
100 sc1.validateParameters({"a": None, "b": None})
101 sc1.validateParameters(
102 [
103 "a",
104 ]
105 )
106 with self.assertRaises(KeyError):
107 sc1.validateParameters({"a", "c"})
109 def testEquality(self):
110 """Test that StorageClass equality works"""
111 className = "TestImage"
112 sc1 = StorageClass(className, pytype=dict)
113 sc2 = StorageClass(className, pytype=dict)
114 self.assertEqual(sc1, sc2)
115 sc3 = StorageClass(className + "2", pytype=str)
116 self.assertNotEqual(sc1, sc3)
118 # Same StorageClass name but different python type
119 sc4 = StorageClass(className, pytype=str)
120 self.assertNotEqual(sc1, sc4)
122 # Parameters
123 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
124 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
125 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"])
126 self.assertEqual(scp, scp1)
127 self.assertNotEqual(scp, scp2)
129 # Now with components
130 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
131 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
132 self.assertEqual(sc5, sc6)
133 self.assertNotEqual(sc5, sc3)
134 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3})
135 self.assertNotEqual(sc5, sc7)
136 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3})
137 self.assertNotEqual(sc5, sc8)
138 sc9 = StorageClass(
139 "Composite",
140 pytype=PythonType,
141 components={"comp1": sc1, "comp2": sc3},
142 delegate="lsst.daf.butler.Butler",
143 )
144 self.assertNotEqual(sc5, sc9)
146 def testTypeEquality(self):
147 sc1 = StorageClass("Something", pytype=dict)
148 self.assertTrue(sc1.is_type(dict), repr(sc1))
149 self.assertFalse(sc1.is_type(str), repr(sc1))
151 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
152 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2))
154 def testRegistry(self):
155 """Check that storage classes can be created on the fly and stored
156 in a registry."""
157 className = "TestImage"
158 factory = StorageClassFactory()
159 newclass = StorageClass(className, pytype=PythonType)
160 factory.registerStorageClass(newclass)
161 sc = factory.getStorageClass(className)
162 self.assertIsInstance(sc, StorageClass)
163 self.assertEqual(sc.name, className)
164 self.assertFalse(sc.components)
165 self.assertEqual(sc.pytype, PythonType)
166 self.assertIn(sc, factory)
167 newclass2 = StorageClass("Temporary2", pytype=str)
168 self.assertNotIn(newclass2, factory)
169 factory.registerStorageClass(newclass2)
170 self.assertIn(newclass2, factory)
171 self.assertIn("Temporary2", factory)
172 self.assertNotIn("Temporary3", factory)
173 self.assertNotIn({}, factory)
175 # Make sure iterators work.
176 keys = set(factory.keys())
177 self.assertIn("Temporary2", keys)
179 iterkeys = {k for k in factory}
180 self.assertEqual(keys, iterkeys)
182 values = set(factory.values())
183 self.assertIn(sc, values)
184 self.assertEqual(len(factory), len(values))
186 external = {k: v for k, v in factory.items()}
187 self.assertIn("Temporary2", external)
189 # Make sure we can't register a storage class with the same name
190 # but different values
191 newclass3 = StorageClass("Temporary2", pytype=dict)
192 with self.assertRaises(ValueError) as cm:
193 factory.registerStorageClass(newclass3)
194 self.assertIn("pytype='dict'", str(cm.exception))
195 with self.assertRaises(ValueError) as cm:
196 factory.registerStorageClass(newclass3, msg="error string")
197 self.assertIn("error string", str(cm.exception))
199 factory._unregisterStorageClass(newclass3.name)
200 self.assertNotIn(newclass3, factory)
201 self.assertNotIn(newclass3.name, factory)
202 factory.registerStorageClass(newclass3)
203 self.assertIn(newclass3, factory)
204 self.assertIn(newclass3.name, factory)
206 # Check you can silently insert something that is already there
207 factory.registerStorageClass(newclass3)
209 def testFactoryConfig(self):
210 factory = StorageClassFactory()
211 factory.addFromConfig(StorageClassConfig())
212 image = factory.getStorageClass("Image")
213 imageF = factory.getStorageClass("ImageF")
214 self.assertIsInstance(imageF, type(image))
215 self.assertNotEqual(imageF, image)
217 # Check component inheritance
218 exposure = factory.getStorageClass("Exposure")
219 exposureF = factory.getStorageClass("ExposureF")
220 self.assertIsInstance(exposureF, type(exposure))
221 self.assertIsInstance(exposure.components["image"], type(image))
222 self.assertNotIsInstance(exposure.components["image"], type(imageF))
223 self.assertIsInstance(exposureF.components["image"], type(image))
224 self.assertIsInstance(exposureF.components["image"], type(imageF))
225 self.assertIn("wcs", exposure.components)
226 self.assertIn("wcs", exposureF.components)
228 # Check parameters
229 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
230 thing1 = factory.getStorageClass("ThingOne")
231 thing2 = factory.getStorageClass("ThingTwo")
232 self.assertIsInstance(thing2, type(thing1))
233 param1 = thing1.parameters
234 param2 = thing2.parameters
235 self.assertIn("param3", thing2.parameters)
236 self.assertNotIn("param3", thing1.parameters)
237 param2.remove("param3")
238 self.assertEqual(param1, param2)
240 # Check that we can't have a new StorageClass that does not
241 # inherit from StorageClass
242 with self.assertRaises(ValueError):
243 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory)
245 sc = factory.makeNewStorageClass("ClassName")
246 self.assertIsInstance(sc(), StorageClass)
248 def testPickle(self):
249 """Test that we can pickle storageclasses."""
250 className = "TestImage"
251 sc = StorageClass(className, pytype=dict)
252 self.assertIsInstance(sc, StorageClass)
253 self.assertEqual(sc.name, className)
254 self.assertFalse(sc.components)
255 sc2 = pickle.loads(pickle.dumps(sc))
256 self.assertEqual(sc2, sc)
258 @classmethod
259 def _convert_type(cls, data):
260 # Test helper function. Fail if the list is empty.
261 if not len(data):
262 raise RuntimeError("Deliberate failure.")
263 return {"key": data}
265 def testConverters(self):
266 """Test conversion maps."""
268 className = "TestConverters"
269 converters = {
270 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict",
271 # Add some entries that will fail to import.
272 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics",
273 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function",
274 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__",
275 "list": get_full_type_name(self._convert_type),
276 }
277 sc = StorageClass(className, pytype=dict, converters=converters)
278 self.assertEqual(len(sc.converters), 5) # Pre-filtering
279 sc2 = StorageClass("Test2", pytype=set)
280 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample")
282 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc))
283 # Initially the converter list is not filtered.
284 self.assertIn("lsst.daf.butler.bad.type", repr(sc))
285 self.assertNotIn("converters", repr(sc2))
287 self.assertTrue(sc.can_convert(sc))
288 self.assertFalse(sc.can_convert(sc2))
289 self.assertTrue(sc.can_convert(sc3))
291 # After we've processed the converters the bad ones will no longer
292 # be reported.
293 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc))
294 self.assertEqual(len(sc.converters), 2)
296 self.assertIsNone(sc.coerce_type(None))
298 converted = sc.coerce_type([1, 2, 3])
299 self.assertEqual(converted, {"key": [1, 2, 3]})
301 # Convert Metrics using a named method converter.
302 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"})
303 converted = sc.coerce_type(metric)
304 self.assertEqual(converted["data"], [1, 2], converted)
306 # Check that python types matching is allowed.
307 sc4 = StorageClass("Test2", pytype=set)
308 self.assertTrue(sc2.can_convert(sc4))
309 converted = sc2.coerce_type({1, 2})
310 self.assertEqual(converted, {1, 2})
312 # Try to coerce a type that is not supported.
313 with self.assertRaises(TypeError):
314 sc.coerce_type(set([1, 2, 3]))
316 # Coerce something that will fail to convert.
317 with self.assertLogs(level=logging.ERROR) as cm:
318 with self.assertRaises(RuntimeError):
319 sc.coerce_type([])
320 self.assertIn("failed to convert type list", cm.output[0])
323if __name__ == "__main__": 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true
324 unittest.main()