Coverage for tests/test_storageClass.py: 12%
224 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-14 19:21 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-14 19:21 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import logging
23import os
24import pickle
25import unittest
27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory
28from lsst.daf.butler.tests import MetricsExample
29from lsst.utils.introspection import get_full_type_name
31"""Tests related to the StorageClass infrastructure.
32"""
34TESTDIR = os.path.abspath(os.path.dirname(__file__))
37class PythonType:
38 """A dummy class to test the registry of Python types."""
40 pass
43class PythonType2:
44 """A dummy class to test the registry of Python types."""
46 pass
49class PythonType3:
50 """A dummy class to test the registry of Python types."""
52 pass
55class NotCopyable:
56 """Class with deep copying disabled."""
58 def __deepcopy__(self, memo=None):
59 raise RuntimeError("Can not be copied.")
62class StorageClassFactoryTestCase(unittest.TestCase):
63 """Tests of the storage class infrastructure."""
65 def testCreation(self):
66 """Test that we can dynamically create storage class subclasses.
68 This is critical for testing the factory functions.
69 """
70 className = "TestImage"
71 sc = StorageClass(className, pytype=dict)
72 self.assertIsInstance(sc, StorageClass)
73 self.assertEqual(sc.name, className)
74 self.assertEqual(str(sc), className)
75 self.assertFalse(sc.components)
76 self.assertTrue(sc.validateInstance({}))
77 self.assertFalse(sc.validateInstance(""))
79 r = repr(sc)
80 self.assertIn("StorageClass", r)
81 self.assertIn(className, r)
82 self.assertNotIn("parameters", r)
83 self.assertIn("pytype='dict'", r)
85 # Ensure we do not have a delegate
86 with self.assertRaises(TypeError):
87 sc.delegate()
89 # Allow no definition of python type
90 scn = StorageClass(className)
91 self.assertIs(scn.pytype, object)
93 # Include some components
94 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc})
95 self.assertIn("comp1", scc.components)
96 r = repr(scc)
97 self.assertIn("comp1", r)
98 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r)
100 # Ensure that we have a delegate
101 self.assertIsInstance(scc.delegate(), StorageClassDelegate)
103 # Check that delegate copy() works.
104 list1 = [1, 2, 3]
105 list2 = scc.delegate().copy(list1)
106 self.assertEqual(list1, list2)
107 list2.append(4)
108 self.assertNotEqual(list1, list2)
110 with self.assertRaises(NotImplementedError):
111 scc.delegate().copy(NotCopyable())
113 # Check we can create a storageClass using the name of an importable
114 # type.
115 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
116 self.assertIsInstance(sc2.pytype(), StorageClassFactory)
117 self.assertIn("butler.core", repr(sc2))
119 def testParameters(self):
120 """Test that we can set parameters and validate them"""
121 pt = ("a", "b")
122 ps = {"a", "b"}
123 pl = ["a", "b"]
124 for p in (pt, ps, pl):
125 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p)
126 self.assertEqual(sc1.parameters, ps)
127 sc1.validateParameters(p)
129 sc1.validateParameters()
130 sc1.validateParameters({"a": None, "b": None})
131 sc1.validateParameters(
132 [
133 "a",
134 ]
135 )
136 with self.assertRaises(KeyError):
137 sc1.validateParameters({"a", "c"})
139 def testEquality(self):
140 """Test that StorageClass equality works"""
141 className = "TestImage"
142 sc1 = StorageClass(className, pytype=dict)
143 sc2 = StorageClass(className, pytype=dict)
144 self.assertEqual(sc1, sc2)
145 sc3 = StorageClass(className + "2", pytype=str)
146 self.assertNotEqual(sc1, sc3)
148 # Same StorageClass name but different python type
149 sc4 = StorageClass(className, pytype=str)
150 self.assertNotEqual(sc1, sc4)
152 # Parameters
153 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
154 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
155 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"])
156 self.assertEqual(scp, scp1)
157 self.assertNotEqual(scp, scp2)
159 # Now with components
160 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
161 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
162 self.assertEqual(sc5, sc6)
163 self.assertNotEqual(sc5, sc3)
164 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3})
165 self.assertNotEqual(sc5, sc7)
166 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3})
167 self.assertNotEqual(sc5, sc8)
168 sc9 = StorageClass(
169 "Composite",
170 pytype=PythonType,
171 components={"comp1": sc1, "comp2": sc3},
172 delegate="lsst.daf.butler.Butler",
173 )
174 self.assertNotEqual(sc5, sc9)
176 def testTypeEquality(self):
177 sc1 = StorageClass("Something", pytype=dict)
178 self.assertTrue(sc1.is_type(dict), repr(sc1))
179 self.assertFalse(sc1.is_type(str), repr(sc1))
181 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory")
182 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2))
184 def testRegistry(self):
185 """Check that storage classes can be created on the fly and stored
186 in a registry.
187 """
188 className = "TestImage"
189 factory = StorageClassFactory()
190 newclass = StorageClass(className, pytype=PythonType)
191 factory.registerStorageClass(newclass)
192 sc = factory.getStorageClass(className)
193 self.assertIsInstance(sc, StorageClass)
194 self.assertEqual(sc.name, className)
195 self.assertFalse(sc.components)
196 self.assertEqual(sc.pytype, PythonType)
197 self.assertIn(sc, factory)
198 newclass2 = StorageClass("Temporary2", pytype=str)
199 self.assertNotIn(newclass2, factory)
200 factory.registerStorageClass(newclass2)
201 self.assertIn(newclass2, factory)
202 self.assertIn("Temporary2", factory)
203 self.assertNotIn("Temporary3", factory)
204 self.assertNotIn({}, factory)
206 # Make sure iterators work.
207 keys = set(factory.keys())
208 self.assertIn("Temporary2", keys)
210 iterkeys = {k for k in factory}
211 self.assertEqual(keys, iterkeys)
213 values = set(factory.values())
214 self.assertIn(sc, values)
215 self.assertEqual(len(factory), len(values))
217 external = {k: v for k, v in factory.items()}
218 self.assertIn("Temporary2", external)
220 # Make sure we can't register a storage class with the same name
221 # but different values
222 newclass3 = StorageClass("Temporary2", pytype=dict)
223 with self.assertRaises(ValueError) as cm:
224 factory.registerStorageClass(newclass3)
225 self.assertIn("pytype='dict'", str(cm.exception))
226 with self.assertRaises(ValueError) as cm:
227 factory.registerStorageClass(newclass3, msg="error string")
228 self.assertIn("error string", str(cm.exception))
230 factory._unregisterStorageClass(newclass3.name)
231 self.assertNotIn(newclass3, factory)
232 self.assertNotIn(newclass3.name, factory)
233 factory.registerStorageClass(newclass3)
234 self.assertIn(newclass3, factory)
235 self.assertIn(newclass3.name, factory)
237 # Check you can silently insert something that is already there
238 factory.registerStorageClass(newclass3)
240 def testFactoryFind(self):
241 # Finding a storage class can involve doing lots of slow imports so
242 # this is a separate test.
243 factory = StorageClassFactory()
244 className = "PythonType3"
245 newclass = StorageClass(className, pytype=PythonType3)
246 factory.registerStorageClass(newclass)
247 sc = factory.getStorageClass(className)
249 # Can we find a storage class from a type.
250 new_sc = factory.findStorageClass(PythonType3)
251 self.assertEqual(new_sc, sc)
253 # Now with slow mode
254 new_sc = factory.findStorageClass(PythonType3, compare_types=True)
255 self.assertEqual(new_sc, sc)
257 # This class will never match.
258 with self.assertRaises(KeyError):
259 factory.findStorageClass(PythonType2, compare_types=True)
261 # Check builtins.
262 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict"))
264 def testFactoryConfig(self):
265 factory = StorageClassFactory()
266 factory.addFromConfig(StorageClassConfig())
267 image = factory.getStorageClass("Image")
268 imageF = factory.getStorageClass("ImageF")
269 self.assertIsInstance(imageF, type(image))
270 self.assertNotEqual(imageF, image)
272 # Check component inheritance
273 exposure = factory.getStorageClass("Exposure")
274 exposureF = factory.getStorageClass("ExposureF")
275 self.assertIsInstance(exposureF, type(exposure))
276 self.assertIsInstance(exposure.components["image"], type(image))
277 self.assertNotIsInstance(exposure.components["image"], type(imageF))
278 self.assertIsInstance(exposureF.components["image"], type(image))
279 self.assertIsInstance(exposureF.components["image"], type(imageF))
280 self.assertIn("wcs", exposure.components)
281 self.assertIn("wcs", exposureF.components)
283 # Check parameters
284 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
285 thing1 = factory.getStorageClass("ThingOne")
286 thing2 = factory.getStorageClass("ThingTwo")
287 self.assertIsInstance(thing2, type(thing1))
288 param1 = thing1.parameters
289 param2 = thing2.parameters
290 self.assertIn("param3", thing2.parameters)
291 self.assertNotIn("param3", thing1.parameters)
292 param2.remove("param3")
293 self.assertEqual(param1, param2)
295 # Check that we can't have a new StorageClass that does not
296 # inherit from StorageClass
297 with self.assertRaises(ValueError):
298 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory)
300 sc = factory.makeNewStorageClass("ClassName")
301 self.assertIsInstance(sc(), StorageClass)
303 def testPickle(self):
304 """Test that we can pickle storageclasses."""
305 className = "TestImage"
306 sc = StorageClass(className, pytype=dict)
307 self.assertIsInstance(sc, StorageClass)
308 self.assertEqual(sc.name, className)
309 self.assertFalse(sc.components)
310 sc2 = pickle.loads(pickle.dumps(sc))
311 self.assertEqual(sc2, sc)
313 @classmethod
314 def _convert_type(cls, data):
315 # Test helper function. Fail if the list is empty.
316 if not len(data):
317 raise RuntimeError("Deliberate failure.")
318 return {"key": data}
320 def testConverters(self):
321 """Test conversion maps."""
322 className = "TestConverters"
323 converters = {
324 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict",
325 # Add some entries that will fail to import.
326 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics",
327 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function",
328 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__",
329 "list": get_full_type_name(self._convert_type),
330 }
331 sc = StorageClass(className, pytype=dict, converters=converters)
332 self.assertEqual(len(sc.converters), 5) # Pre-filtering
333 sc2 = StorageClass("Test2", pytype=set)
334 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample")
336 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc))
337 # Initially the converter list is not filtered.
338 self.assertIn("lsst.daf.butler.bad.type", repr(sc))
339 self.assertNotIn("converters", repr(sc2))
341 self.assertTrue(sc.can_convert(sc))
342 self.assertFalse(sc.can_convert(sc2))
343 self.assertTrue(sc.can_convert(sc3))
345 # After we've processed the converters the bad ones will no longer
346 # be reported.
347 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc))
348 self.assertEqual(len(sc.converters), 2)
350 self.assertIsNone(sc.coerce_type(None))
352 converted = sc.coerce_type([1, 2, 3])
353 self.assertEqual(converted, {"key": [1, 2, 3]})
355 # Convert Metrics using a named method converter.
356 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"})
357 converted = sc.coerce_type(metric)
358 self.assertEqual(converted["data"], [1, 2], converted)
360 # Check that python types matching is allowed.
361 sc4 = StorageClass("Test2", pytype=set)
362 self.assertTrue(sc2.can_convert(sc4))
363 converted = sc2.coerce_type({1, 2})
364 self.assertEqual(converted, {1, 2})
366 # Try to coerce a type that is not supported.
367 with self.assertRaises(TypeError):
368 sc.coerce_type({1, 2, 3})
370 # Coerce something that will fail to convert.
371 with self.assertLogs(level=logging.ERROR) as cm:
372 with self.assertRaises(RuntimeError):
373 sc.coerce_type([])
374 self.assertIn("failed to convert type list", cm.output[0])
377if __name__ == "__main__":
378 unittest.main()