Coverage for tests/test_storageClass.py: 12%
224 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
28import logging
29import os
30import pickle
31import unittest
33from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory
34from lsst.daf.butler.tests import MetricsExample
35from lsst.utils.introspection import get_full_type_name
37"""Tests related to the StorageClass infrastructure.
38"""
40TESTDIR = os.path.abspath(os.path.dirname(__file__))
43class PythonType:
44 """A dummy class to test the registry of Python types."""
46 pass
49class PythonType2:
50 """A dummy class to test the registry of Python types."""
52 pass
55class PythonType3:
56 """A dummy class to test the registry of Python types."""
58 pass
61class NotCopyable:
62 """Class with deep copying disabled."""
64 def __deepcopy__(self, memo=None):
65 raise RuntimeError("Can not be copied.")
68class StorageClassFactoryTestCase(unittest.TestCase):
69 """Tests of the storage class infrastructure."""
71 def testCreation(self):
72 """Test that we can dynamically create storage class subclasses.
74 This is critical for testing the factory functions.
75 """
76 className = "TestImage"
77 sc = StorageClass(className, pytype=dict)
78 self.assertIsInstance(sc, StorageClass)
79 self.assertEqual(sc.name, className)
80 self.assertEqual(str(sc), className)
81 self.assertFalse(sc.components)
82 self.assertTrue(sc.validateInstance({}))
83 self.assertFalse(sc.validateInstance(""))
85 r = repr(sc)
86 self.assertIn("StorageClass", r)
87 self.assertIn(className, r)
88 self.assertNotIn("parameters", r)
89 self.assertIn("pytype='dict'", r)
91 # Ensure we do not have a delegate
92 with self.assertRaises(TypeError):
93 sc.delegate()
95 # Allow no definition of python type
96 scn = StorageClass(className)
97 self.assertIs(scn.pytype, object)
99 # Include some components
100 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc})
101 self.assertIn("comp1", scc.components)
102 r = repr(scc)
103 self.assertIn("comp1", r)
104 self.assertIn("lsst.daf.butler.StorageClassDelegate", r)
106 # Ensure that we have a delegate
107 self.assertIsInstance(scc.delegate(), StorageClassDelegate)
109 # Check that delegate copy() works.
110 list1 = [1, 2, 3]
111 list2 = scc.delegate().copy(list1)
112 self.assertEqual(list1, list2)
113 list2.append(4)
114 self.assertNotEqual(list1, list2)
116 with self.assertRaises(NotImplementedError):
117 scc.delegate().copy(NotCopyable())
119 # Check we can create a storageClass using the name of an importable
120 # type.
121 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory")
122 self.assertIsInstance(sc2.pytype(), StorageClassFactory)
123 self.assertIn("butler", repr(sc2))
125 def testParameters(self):
126 """Test that we can set parameters and validate them"""
127 pt = ("a", "b")
128 ps = {"a", "b"}
129 pl = ["a", "b"]
130 for p in (pt, ps, pl):
131 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p)
132 self.assertEqual(sc1.parameters, ps)
133 sc1.validateParameters(p)
135 sc1.validateParameters()
136 sc1.validateParameters({"a": None, "b": None})
137 sc1.validateParameters(
138 [
139 "a",
140 ]
141 )
142 with self.assertRaises(KeyError):
143 sc1.validateParameters({"a", "c"})
145 def testEquality(self):
146 """Test that StorageClass equality works"""
147 className = "TestImage"
148 sc1 = StorageClass(className, pytype=dict)
149 sc2 = StorageClass(className, pytype=dict)
150 self.assertEqual(sc1, sc2)
151 sc3 = StorageClass(className + "2", pytype=str)
152 self.assertNotEqual(sc1, sc3)
154 # Same StorageClass name but different python type
155 sc4 = StorageClass(className, pytype=str)
156 self.assertNotEqual(sc1, sc4)
158 # Parameters
159 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
160 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"])
161 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"])
162 self.assertEqual(scp, scp1)
163 self.assertNotEqual(scp, scp2)
165 # Now with components
166 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
167 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3})
168 self.assertEqual(sc5, sc6)
169 self.assertNotEqual(sc5, sc3)
170 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3})
171 self.assertNotEqual(sc5, sc7)
172 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3})
173 self.assertNotEqual(sc5, sc8)
174 sc9 = StorageClass(
175 "Composite",
176 pytype=PythonType,
177 components={"comp1": sc1, "comp2": sc3},
178 delegate="lsst.daf.butler.Butler",
179 )
180 self.assertNotEqual(sc5, sc9)
182 def testTypeEquality(self):
183 sc1 = StorageClass("Something", pytype=dict)
184 self.assertTrue(sc1.is_type(dict), repr(sc1))
185 self.assertFalse(sc1.is_type(str), repr(sc1))
187 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory")
188 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2))
190 def testRegistry(self):
191 """Check that storage classes can be created on the fly and stored
192 in a registry.
193 """
194 className = "TestImage"
195 factory = StorageClassFactory()
196 newclass = StorageClass(className, pytype=PythonType)
197 factory.registerStorageClass(newclass)
198 sc = factory.getStorageClass(className)
199 self.assertIsInstance(sc, StorageClass)
200 self.assertEqual(sc.name, className)
201 self.assertFalse(sc.components)
202 self.assertEqual(sc.pytype, PythonType)
203 self.assertIn(sc, factory)
204 newclass2 = StorageClass("Temporary2", pytype=str)
205 self.assertNotIn(newclass2, factory)
206 factory.registerStorageClass(newclass2)
207 self.assertIn(newclass2, factory)
208 self.assertIn("Temporary2", factory)
209 self.assertNotIn("Temporary3", factory)
210 self.assertNotIn({}, factory)
212 # Make sure iterators work.
213 keys = set(factory.keys())
214 self.assertIn("Temporary2", keys)
216 iterkeys = set(factory)
217 self.assertEqual(keys, iterkeys)
219 values = set(factory.values())
220 self.assertIn(sc, values)
221 self.assertEqual(len(factory), len(values))
223 external = dict(factory.items())
224 self.assertIn("Temporary2", external)
226 # Make sure we can't register a storage class with the same name
227 # but different values
228 newclass3 = StorageClass("Temporary2", pytype=dict)
229 with self.assertRaises(ValueError) as cm:
230 factory.registerStorageClass(newclass3)
231 self.assertIn("pytype='dict'", str(cm.exception))
232 with self.assertRaises(ValueError) as cm:
233 factory.registerStorageClass(newclass3, msg="error string")
234 self.assertIn("error string", str(cm.exception))
236 factory._unregisterStorageClass(newclass3.name)
237 self.assertNotIn(newclass3, factory)
238 self.assertNotIn(newclass3.name, factory)
239 factory.registerStorageClass(newclass3)
240 self.assertIn(newclass3, factory)
241 self.assertIn(newclass3.name, factory)
243 # Check you can silently insert something that is already there
244 factory.registerStorageClass(newclass3)
246 def testFactoryFind(self):
247 # Finding a storage class can involve doing lots of slow imports so
248 # this is a separate test.
249 factory = StorageClassFactory()
250 className = "PythonType3"
251 newclass = StorageClass(className, pytype=PythonType3)
252 factory.registerStorageClass(newclass)
253 sc = factory.getStorageClass(className)
255 # Can we find a storage class from a type.
256 new_sc = factory.findStorageClass(PythonType3)
257 self.assertEqual(new_sc, sc)
259 # Now with slow mode
260 new_sc = factory.findStorageClass(PythonType3, compare_types=True)
261 self.assertEqual(new_sc, sc)
263 # This class will never match.
264 with self.assertRaises(KeyError):
265 factory.findStorageClass(PythonType2, compare_types=True)
267 # Check builtins.
268 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict"))
270 def testFactoryConfig(self):
271 factory = StorageClassFactory()
272 factory.addFromConfig(StorageClassConfig())
273 image = factory.getStorageClass("Image")
274 imageF = factory.getStorageClass("ImageF")
275 self.assertIsInstance(imageF, type(image))
276 self.assertNotEqual(imageF, image)
278 # Check component inheritance
279 exposure = factory.getStorageClass("Exposure")
280 exposureF = factory.getStorageClass("ExposureF")
281 self.assertIsInstance(exposureF, type(exposure))
282 self.assertIsInstance(exposure.components["image"], type(image))
283 self.assertNotIsInstance(exposure.components["image"], type(imageF))
284 self.assertIsInstance(exposureF.components["image"], type(image))
285 self.assertIsInstance(exposureF.components["image"], type(imageF))
286 self.assertIn("wcs", exposure.components)
287 self.assertIn("wcs", exposureF.components)
289 # Check parameters
290 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml"))
291 thing1 = factory.getStorageClass("ThingOne")
292 thing2 = factory.getStorageClass("ThingTwo")
293 self.assertIsInstance(thing2, type(thing1))
294 param1 = thing1.parameters
295 param2 = thing2.parameters
296 self.assertIn("param3", thing2.parameters)
297 self.assertNotIn("param3", thing1.parameters)
298 param2.remove("param3")
299 self.assertEqual(param1, param2)
301 # Check that we can't have a new StorageClass that does not
302 # inherit from StorageClass
303 with self.assertRaises(ValueError):
304 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory)
306 sc = factory.makeNewStorageClass("ClassName")
307 self.assertIsInstance(sc(), StorageClass)
309 def testPickle(self):
310 """Test that we can pickle storageclasses."""
311 className = "TestImage"
312 sc = StorageClass(className, pytype=dict)
313 self.assertIsInstance(sc, StorageClass)
314 self.assertEqual(sc.name, className)
315 self.assertFalse(sc.components)
316 sc2 = pickle.loads(pickle.dumps(sc))
317 self.assertEqual(sc2, sc)
319 @classmethod
320 def _convert_type(cls, data):
321 # Test helper function. Fail if the list is empty.
322 if not len(data):
323 raise RuntimeError("Deliberate failure.")
324 return {"key": data}
326 def testConverters(self):
327 """Test conversion maps."""
328 className = "TestConverters"
329 converters = {
330 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict",
331 # Add some entries that will fail to import.
332 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics",
333 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function",
334 "lsst.daf.butler.Butler": "lsst.daf.butler.location.__all__",
335 "list": get_full_type_name(self._convert_type),
336 }
337 sc = StorageClass(className, pytype=dict, converters=converters)
338 self.assertEqual(len(sc.converters), 5) # Pre-filtering
339 sc2 = StorageClass("Test2", pytype=set)
340 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample")
342 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc))
343 # Initially the converter list is not filtered.
344 self.assertIn("lsst.daf.butler.bad.type", repr(sc))
345 self.assertNotIn("converters", repr(sc2))
347 self.assertTrue(sc.can_convert(sc))
348 self.assertFalse(sc.can_convert(sc2))
349 self.assertTrue(sc.can_convert(sc3))
351 # After we've processed the converters the bad ones will no longer
352 # be reported.
353 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc))
354 self.assertEqual(len(sc.converters), 2)
356 self.assertIsNone(sc.coerce_type(None))
358 converted = sc.coerce_type([1, 2, 3])
359 self.assertEqual(converted, {"key": [1, 2, 3]})
361 # Convert Metrics using a named method converter.
362 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"})
363 converted = sc.coerce_type(metric)
364 self.assertEqual(converted["data"], [1, 2], converted)
366 # Check that python types matching is allowed.
367 sc4 = StorageClass("Test2", pytype=set)
368 self.assertTrue(sc2.can_convert(sc4))
369 converted = sc2.coerce_type({1, 2})
370 self.assertEqual(converted, {1, 2})
372 # Try to coerce a type that is not supported.
373 with self.assertRaises(TypeError):
374 sc.coerce_type({1, 2, 3})
376 # Coerce something that will fail to convert.
377 with self.assertLogs(level=logging.ERROR) as cm:
378 with self.assertRaises(RuntimeError):
379 sc.coerce_type([])
380 self.assertIn("failed to convert type list", cm.output[0])
383if __name__ == "__main__":
384 unittest.main()