Coverage for tests/test_storageClass.py: 12%
224 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-02 02:16 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-02 02:16 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24import pickle
25import unittest
27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory
28from lsst.daf.butler.tests import MetricsExample
29from lsst.utils.introspection import get_full_type_name
31"""Tests related to the StorageClass infrastructure.
32"""
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class PythonType:
38 """A dummy class to test the registry of Python types."""
40 pass
43class PythonType2:
44 """A dummy class to test the registry of Python types."""
46 pass
49class PythonType3:
50 """A dummy class to test the registry of Python types."""
52 pass
55class NotCopyable:
56 def __deepcopy__(self, memo=None):
57 raise RuntimeError("Can not be copied.")
60class StorageClassFactoryTestCase(unittest.TestCase):
61 """Tests of the storage class infrastructure."""
63 def testCreation(self):
64 """Test that we can dynamically create storage class subclasses.
66 This is critical for testing the factory functions."""
67 className = "TestImage"
68 sc = StorageClass(className, pytype=dict)
69 self.assertIsInstance(sc, StorageClass)
70 self.assertEqual(sc.name, className)
71 self.assertEqual(str(sc), className)
72 self.assertFalse(sc.components)
73 self.assertTrue(sc.validateInstance({}))
74 self.assertFalse(sc.validateInstance(""))
76 r = repr(sc)
77 self.assertIn("StorageClass", r)
78 self.assertIn(className, r)
79 self.assertNotIn("parameters", r)
80 self.assertIn("pytype='dict'", r)
82 # Ensure we do not have a delegate
83 with self.assertRaises(TypeError):
84 sc.delegate()
86 # Allow no definition of python type
87 scn = StorageClass(className)
88 self.assertIs(scn.pytype, object)
90 # Include some components
91 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc})
92 self.assertIn("comp1", scc.components)
93 r = repr(scc)
94 self.assertIn("comp1", r)
95 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r)
97 # Ensure that we have a delegate
98 self.assertIsInstance(scc.delegate(), StorageClassDelegate)
100 # Check that delegate copy() works.
101 list1 = [1, 2, 3]
102 list2 = scc.delegate().copy(list1)
103 self.assertEqual(list1, list2)
104 list2.append(4)
105 self.assertNotEqual(list1, list2)
107 with self.assertRaises(NotImplementedError):
108 scc.delegate().copy(NotCopyable())
110 # Check we can create a storageClass using the name of an importable
111 # type.
112 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
113 self.assertIsInstance(sc2.pytype(), StorageClassFactory)
114 self.assertIn("butler.core", repr(sc2))
116 def testParameters(self):
117 """Test that we can set parameters and validate them"""
118 pt = ("a", "b")
119 ps = {"a", "b"}
120 pl = ["a", "b"]
121 for p in (pt, ps, pl):
122 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p)
123 self.assertEqual(sc1.parameters, ps)
124 sc1.validateParameters(p)
126 sc1.validateParameters()
127 sc1.validateParameters({"a": None, "b": None})
128 sc1.validateParameters(
129 [
130 "a",
131 ]
132 )
133 with self.assertRaises(KeyError):
134 sc1.validateParameters({"a", "c"})
136 def testEquality(self):
137 """Test that StorageClass equality works"""
138 className = "TestImage"
139 sc1 = StorageClass(className, pytype=dict)
140 sc2 = StorageClass(className, pytype=dict)
141 self.assertEqual(sc1, sc2)
142 sc3 = StorageClass(className + "2", pytype=str)
143 self.assertNotEqual(sc1, sc3)
145 # Same StorageClass name but different python type
146 sc4 = StorageClass(className, pytype=str)
147 self.assertNotEqual(sc1, sc4)
149 # Parameters
150 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
151 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
152 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"])
153 self.assertEqual(scp, scp1)
154 self.assertNotEqual(scp, scp2)
156 # Now with components
157 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
158 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
159 self.assertEqual(sc5, sc6)
160 self.assertNotEqual(sc5, sc3)
161 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3})
162 self.assertNotEqual(sc5, sc7)
163 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3})
164 self.assertNotEqual(sc5, sc8)
165 sc9 = StorageClass(
166 "Composite",
167 pytype=PythonType,
168 components={"comp1": sc1, "comp2": sc3},
169 delegate="lsst.daf.butler.Butler",
170 )
171 self.assertNotEqual(sc5, sc9)
173 def testTypeEquality(self):
174 sc1 = StorageClass("Something", pytype=dict)
175 self.assertTrue(sc1.is_type(dict), repr(sc1))
176 self.assertFalse(sc1.is_type(str), repr(sc1))
178 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
179 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2))
181 def testRegistry(self):
182 """Check that storage classes can be created on the fly and stored
183 in a registry."""
184 className = "TestImage"
185 factory = StorageClassFactory()
186 newclass = StorageClass(className, pytype=PythonType)
187 factory.registerStorageClass(newclass)
188 sc = factory.getStorageClass(className)
189 self.assertIsInstance(sc, StorageClass)
190 self.assertEqual(sc.name, className)
191 self.assertFalse(sc.components)
192 self.assertEqual(sc.pytype, PythonType)
193 self.assertIn(sc, factory)
194 newclass2 = StorageClass("Temporary2", pytype=str)
195 self.assertNotIn(newclass2, factory)
196 factory.registerStorageClass(newclass2)
197 self.assertIn(newclass2, factory)
198 self.assertIn("Temporary2", factory)
199 self.assertNotIn("Temporary3", factory)
200 self.assertNotIn({}, factory)
202 # Make sure iterators work.
203 keys = set(factory.keys())
204 self.assertIn("Temporary2", keys)
206 iterkeys = {k for k in factory}
207 self.assertEqual(keys, iterkeys)
209 values = set(factory.values())
210 self.assertIn(sc, values)
211 self.assertEqual(len(factory), len(values))
213 external = {k: v for k, v in factory.items()}
214 self.assertIn("Temporary2", external)
216 # Make sure we can't register a storage class with the same name
217 # but different values
218 newclass3 = StorageClass("Temporary2", pytype=dict)
219 with self.assertRaises(ValueError) as cm:
220 factory.registerStorageClass(newclass3)
221 self.assertIn("pytype='dict'", str(cm.exception))
222 with self.assertRaises(ValueError) as cm:
223 factory.registerStorageClass(newclass3, msg="error string")
224 self.assertIn("error string", str(cm.exception))
226 factory._unregisterStorageClass(newclass3.name)
227 self.assertNotIn(newclass3, factory)
228 self.assertNotIn(newclass3.name, factory)
229 factory.registerStorageClass(newclass3)
230 self.assertIn(newclass3, factory)
231 self.assertIn(newclass3.name, factory)
233 # Check you can silently insert something that is already there
234 factory.registerStorageClass(newclass3)
236 def testFactoryFind(self):
237 # Finding a storage class can involve doing lots of slow imports so
238 # this is a separate test.
239 factory = StorageClassFactory()
240 className = "PythonType3"
241 newclass = StorageClass(className, pytype=PythonType3)
242 factory.registerStorageClass(newclass)
243 sc = factory.getStorageClass(className)
245 # Can we find a storage class from a type.
246 new_sc = factory.findStorageClass(PythonType3)
247 self.assertEqual(new_sc, sc)
249 # Now with slow mode
250 new_sc = factory.findStorageClass(PythonType3, compare_types=True)
251 self.assertEqual(new_sc, sc)
253 # This class will never match.
254 with self.assertRaises(KeyError):
255 factory.findStorageClass(PythonType2, compare_types=True)
257 # Check builtins.
258 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict"))
260 def testFactoryConfig(self):
261 factory = StorageClassFactory()
262 factory.addFromConfig(StorageClassConfig())
263 image = factory.getStorageClass("Image")
264 imageF = factory.getStorageClass("ImageF")
265 self.assertIsInstance(imageF, type(image))
266 self.assertNotEqual(imageF, image)
268 # Check component inheritance
269 exposure = factory.getStorageClass("Exposure")
270 exposureF = factory.getStorageClass("ExposureF")
271 self.assertIsInstance(exposureF, type(exposure))
272 self.assertIsInstance(exposure.components["image"], type(image))
273 self.assertNotIsInstance(exposure.components["image"], type(imageF))
274 self.assertIsInstance(exposureF.components["image"], type(image))
275 self.assertIsInstance(exposureF.components["image"], type(imageF))
276 self.assertIn("wcs", exposure.components)
277 self.assertIn("wcs", exposureF.components)
279 # Check parameters
280 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
281 thing1 = factory.getStorageClass("ThingOne")
282 thing2 = factory.getStorageClass("ThingTwo")
283 self.assertIsInstance(thing2, type(thing1))
284 param1 = thing1.parameters
285 param2 = thing2.parameters
286 self.assertIn("param3", thing2.parameters)
287 self.assertNotIn("param3", thing1.parameters)
288 param2.remove("param3")
289 self.assertEqual(param1, param2)
291 # Check that we can't have a new StorageClass that does not
292 # inherit from StorageClass
293 with self.assertRaises(ValueError):
294 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory)
296 sc = factory.makeNewStorageClass("ClassName")
297 self.assertIsInstance(sc(), StorageClass)
299 def testPickle(self):
300 """Test that we can pickle storageclasses."""
301 className = "TestImage"
302 sc = StorageClass(className, pytype=dict)
303 self.assertIsInstance(sc, StorageClass)
304 self.assertEqual(sc.name, className)
305 self.assertFalse(sc.components)
306 sc2 = pickle.loads(pickle.dumps(sc))
307 self.assertEqual(sc2, sc)
309 @classmethod
310 def _convert_type(cls, data):
311 # Test helper function. Fail if the list is empty.
312 if not len(data):
313 raise RuntimeError("Deliberate failure.")
314 return {"key": data}
316 def testConverters(self):
317 """Test conversion maps."""
319 className = "TestConverters"
320 converters = {
321 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict",
322 # Add some entries that will fail to import.
323 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics",
324 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function",
325 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__",
326 "list": get_full_type_name(self._convert_type),
327 }
328 sc = StorageClass(className, pytype=dict, converters=converters)
329 self.assertEqual(len(sc.converters), 5) # Pre-filtering
330 sc2 = StorageClass("Test2", pytype=set)
331 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample")
333 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc))
334 # Initially the converter list is not filtered.
335 self.assertIn("lsst.daf.butler.bad.type", repr(sc))
336 self.assertNotIn("converters", repr(sc2))
338 self.assertTrue(sc.can_convert(sc))
339 self.assertFalse(sc.can_convert(sc2))
340 self.assertTrue(sc.can_convert(sc3))
342 # After we've processed the converters the bad ones will no longer
343 # be reported.
344 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc))
345 self.assertEqual(len(sc.converters), 2)
347 self.assertIsNone(sc.coerce_type(None))
349 converted = sc.coerce_type([1, 2, 3])
350 self.assertEqual(converted, {"key": [1, 2, 3]})
352 # Convert Metrics using a named method converter.
353 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"})
354 converted = sc.coerce_type(metric)
355 self.assertEqual(converted["data"], [1, 2], converted)
357 # Check that python types matching is allowed.
358 sc4 = StorageClass("Test2", pytype=set)
359 self.assertTrue(sc2.can_convert(sc4))
360 converted = sc2.coerce_type({1, 2})
361 self.assertEqual(converted, {1, 2})
363 # Try to coerce a type that is not supported.
364 with self.assertRaises(TypeError):
365 sc.coerce_type(set([1, 2, 3]))
367 # Coerce something that will fail to convert.
368 with self.assertLogs(level=logging.ERROR) as cm:
369 with self.assertRaises(RuntimeError):
370 sc.coerce_type([])
371 self.assertIn("failed to convert type list", cm.output[0])
374if __name__ == "__main__":
375 unittest.main()