Coverage for tests/test_storageClass.py: 11%
214 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-22 02:18 -0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-22 02:18 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24import pickle
25import unittest
27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory
28from lsst.daf.butler.tests import MetricsExample
29from lsst.utils.introspection import get_full_type_name
31"""Tests related to the StorageClass infrastructure.
32"""
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class PythonType:
38 """A dummy class to test the registry of Python types."""
40 pass
43class PythonType2:
44 """A dummy class to test the registry of Python types."""
46 pass
49class PythonType3:
50 """A dummy class to test the registry of Python types."""
52 pass
55class StorageClassFactoryTestCase(unittest.TestCase):
56 """Tests of the storage class infrastructure."""
58 def testCreation(self):
59 """Test that we can dynamically create storage class subclasses.
61 This is critical for testing the factory functions."""
62 className = "TestImage"
63 sc = StorageClass(className, pytype=dict)
64 self.assertIsInstance(sc, StorageClass)
65 self.assertEqual(sc.name, className)
66 self.assertEqual(str(sc), className)
67 self.assertFalse(sc.components)
68 self.assertTrue(sc.validateInstance({}))
69 self.assertFalse(sc.validateInstance(""))
71 r = repr(sc)
72 self.assertIn("StorageClass", r)
73 self.assertIn(className, r)
74 self.assertNotIn("parameters", r)
75 self.assertIn("pytype='dict'", r)
77 # Ensure we do not have a delegate
78 with self.assertRaises(TypeError):
79 sc.delegate()
81 # Allow no definition of python type
82 scn = StorageClass(className)
83 self.assertIs(scn.pytype, object)
85 # Include some components
86 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc})
87 self.assertIn("comp1", scc.components)
88 r = repr(scc)
89 self.assertIn("comp1", r)
90 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r)
92 # Ensure that we have a delegate
93 self.assertIsInstance(scc.delegate(), StorageClassDelegate)
95 # Check we can create a storageClass using the name of an importable
96 # type.
97 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
98 self.assertIsInstance(sc2.pytype(), StorageClassFactory)
99 self.assertIn("butler.core", repr(sc2))
101 def testParameters(self):
102 """Test that we can set parameters and validate them"""
103 pt = ("a", "b")
104 ps = {"a", "b"}
105 pl = ["a", "b"]
106 for p in (pt, ps, pl):
107 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p)
108 self.assertEqual(sc1.parameters, ps)
109 sc1.validateParameters(p)
111 sc1.validateParameters()
112 sc1.validateParameters({"a": None, "b": None})
113 sc1.validateParameters(
114 [
115 "a",
116 ]
117 )
118 with self.assertRaises(KeyError):
119 sc1.validateParameters({"a", "c"})
121 def testEquality(self):
122 """Test that StorageClass equality works"""
123 className = "TestImage"
124 sc1 = StorageClass(className, pytype=dict)
125 sc2 = StorageClass(className, pytype=dict)
126 self.assertEqual(sc1, sc2)
127 sc3 = StorageClass(className + "2", pytype=str)
128 self.assertNotEqual(sc1, sc3)
130 # Same StorageClass name but different python type
131 sc4 = StorageClass(className, pytype=str)
132 self.assertNotEqual(sc1, sc4)
134 # Parameters
135 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
136 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
137 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"])
138 self.assertEqual(scp, scp1)
139 self.assertNotEqual(scp, scp2)
141 # Now with components
142 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
143 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
144 self.assertEqual(sc5, sc6)
145 self.assertNotEqual(sc5, sc3)
146 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3})
147 self.assertNotEqual(sc5, sc7)
148 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3})
149 self.assertNotEqual(sc5, sc8)
150 sc9 = StorageClass(
151 "Composite",
152 pytype=PythonType,
153 components={"comp1": sc1, "comp2": sc3},
154 delegate="lsst.daf.butler.Butler",
155 )
156 self.assertNotEqual(sc5, sc9)
158 def testTypeEquality(self):
159 sc1 = StorageClass("Something", pytype=dict)
160 self.assertTrue(sc1.is_type(dict), repr(sc1))
161 self.assertFalse(sc1.is_type(str), repr(sc1))
163 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
164 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2))
166 def testRegistry(self):
167 """Check that storage classes can be created on the fly and stored
168 in a registry."""
169 className = "TestImage"
170 factory = StorageClassFactory()
171 newclass = StorageClass(className, pytype=PythonType)
172 factory.registerStorageClass(newclass)
173 sc = factory.getStorageClass(className)
174 self.assertIsInstance(sc, StorageClass)
175 self.assertEqual(sc.name, className)
176 self.assertFalse(sc.components)
177 self.assertEqual(sc.pytype, PythonType)
178 self.assertIn(sc, factory)
179 newclass2 = StorageClass("Temporary2", pytype=str)
180 self.assertNotIn(newclass2, factory)
181 factory.registerStorageClass(newclass2)
182 self.assertIn(newclass2, factory)
183 self.assertIn("Temporary2", factory)
184 self.assertNotIn("Temporary3", factory)
185 self.assertNotIn({}, factory)
187 # Make sure iterators work.
188 keys = set(factory.keys())
189 self.assertIn("Temporary2", keys)
191 iterkeys = {k for k in factory}
192 self.assertEqual(keys, iterkeys)
194 values = set(factory.values())
195 self.assertIn(sc, values)
196 self.assertEqual(len(factory), len(values))
198 external = {k: v for k, v in factory.items()}
199 self.assertIn("Temporary2", external)
201 # Make sure we can't register a storage class with the same name
202 # but different values
203 newclass3 = StorageClass("Temporary2", pytype=dict)
204 with self.assertRaises(ValueError) as cm:
205 factory.registerStorageClass(newclass3)
206 self.assertIn("pytype='dict'", str(cm.exception))
207 with self.assertRaises(ValueError) as cm:
208 factory.registerStorageClass(newclass3, msg="error string")
209 self.assertIn("error string", str(cm.exception))
211 factory._unregisterStorageClass(newclass3.name)
212 self.assertNotIn(newclass3, factory)
213 self.assertNotIn(newclass3.name, factory)
214 factory.registerStorageClass(newclass3)
215 self.assertIn(newclass3, factory)
216 self.assertIn(newclass3.name, factory)
218 # Check you can silently insert something that is already there
219 factory.registerStorageClass(newclass3)
221 def testFactoryFind(self):
222 # Finding a storage class can involve doing lots of slow imports so
223 # this is a separate test.
224 factory = StorageClassFactory()
225 className = "PythonType3"
226 newclass = StorageClass(className, pytype=PythonType3)
227 factory.registerStorageClass(newclass)
228 sc = factory.getStorageClass(className)
230 # Can we find a storage class from a type.
231 new_sc = factory.findStorageClass(PythonType3)
232 self.assertEqual(new_sc, sc)
234 # Now with slow mode
235 new_sc = factory.findStorageClass(PythonType3, compare_types=True)
236 self.assertEqual(new_sc, sc)
238 # This class will never match.
239 with self.assertRaises(KeyError):
240 factory.findStorageClass(PythonType2, compare_types=True)
242 # Check builtins.
243 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict"))
245 def testFactoryConfig(self):
246 factory = StorageClassFactory()
247 factory.addFromConfig(StorageClassConfig())
248 image = factory.getStorageClass("Image")
249 imageF = factory.getStorageClass("ImageF")
250 self.assertIsInstance(imageF, type(image))
251 self.assertNotEqual(imageF, image)
253 # Check component inheritance
254 exposure = factory.getStorageClass("Exposure")
255 exposureF = factory.getStorageClass("ExposureF")
256 self.assertIsInstance(exposureF, type(exposure))
257 self.assertIsInstance(exposure.components["image"], type(image))
258 self.assertNotIsInstance(exposure.components["image"], type(imageF))
259 self.assertIsInstance(exposureF.components["image"], type(image))
260 self.assertIsInstance(exposureF.components["image"], type(imageF))
261 self.assertIn("wcs", exposure.components)
262 self.assertIn("wcs", exposureF.components)
264 # Check parameters
265 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
266 thing1 = factory.getStorageClass("ThingOne")
267 thing2 = factory.getStorageClass("ThingTwo")
268 self.assertIsInstance(thing2, type(thing1))
269 param1 = thing1.parameters
270 param2 = thing2.parameters
271 self.assertIn("param3", thing2.parameters)
272 self.assertNotIn("param3", thing1.parameters)
273 param2.remove("param3")
274 self.assertEqual(param1, param2)
276 # Check that we can't have a new StorageClass that does not
277 # inherit from StorageClass
278 with self.assertRaises(ValueError):
279 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory)
281 sc = factory.makeNewStorageClass("ClassName")
282 self.assertIsInstance(sc(), StorageClass)
284 def testPickle(self):
285 """Test that we can pickle storageclasses."""
286 className = "TestImage"
287 sc = StorageClass(className, pytype=dict)
288 self.assertIsInstance(sc, StorageClass)
289 self.assertEqual(sc.name, className)
290 self.assertFalse(sc.components)
291 sc2 = pickle.loads(pickle.dumps(sc))
292 self.assertEqual(sc2, sc)
294 @classmethod
295 def _convert_type(cls, data):
296 # Test helper function. Fail if the list is empty.
297 if not len(data):
298 raise RuntimeError("Deliberate failure.")
299 return {"key": data}
301 def testConverters(self):
302 """Test conversion maps."""
304 className = "TestConverters"
305 converters = {
306 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict",
307 # Add some entries that will fail to import.
308 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics",
309 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function",
310 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__",
311 "list": get_full_type_name(self._convert_type),
312 }
313 sc = StorageClass(className, pytype=dict, converters=converters)
314 self.assertEqual(len(sc.converters), 5) # Pre-filtering
315 sc2 = StorageClass("Test2", pytype=set)
316 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample")
318 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc))
319 # Initially the converter list is not filtered.
320 self.assertIn("lsst.daf.butler.bad.type", repr(sc))
321 self.assertNotIn("converters", repr(sc2))
323 self.assertTrue(sc.can_convert(sc))
324 self.assertFalse(sc.can_convert(sc2))
325 self.assertTrue(sc.can_convert(sc3))
327 # After we've processed the converters the bad ones will no longer
328 # be reported.
329 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc))
330 self.assertEqual(len(sc.converters), 2)
332 self.assertIsNone(sc.coerce_type(None))
334 converted = sc.coerce_type([1, 2, 3])
335 self.assertEqual(converted, {"key": [1, 2, 3]})
337 # Convert Metrics using a named method converter.
338 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"})
339 converted = sc.coerce_type(metric)
340 self.assertEqual(converted["data"], [1, 2], converted)
342 # Check that python types matching is allowed.
343 sc4 = StorageClass("Test2", pytype=set)
344 self.assertTrue(sc2.can_convert(sc4))
345 converted = sc2.coerce_type({1, 2})
346 self.assertEqual(converted, {1, 2})
348 # Try to coerce a type that is not supported.
349 with self.assertRaises(TypeError):
350 sc.coerce_type(set([1, 2, 3]))
352 # Coerce something that will fail to convert.
353 with self.assertLogs(level=logging.ERROR) as cm:
354 with self.assertRaises(RuntimeError):
355 sc.coerce_type([])
356 self.assertIn("failed to convert type list", cm.output[0])
359if __name__ == "__main__":
360 unittest.main()