Coverage for tests/test_storageClass.py: 14%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

171 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import pickle 

24import unittest 

25import logging 

26 

27from lsst.utils.introspection import get_full_type_name 

28 

29from lsst.daf.butler import StorageClass, StorageClassFactory, StorageClassConfig, StorageClassDelegate 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 pass 

40 

41 

42class StorageClassFactoryTestCase(unittest.TestCase): 

43 """Tests of the storage class infrastructure. 

44 """ 

45 

46 def testCreation(self): 

47 """Test that we can dynamically create storage class subclasses. 

48 

49 This is critical for testing the factory functions.""" 

50 className = "TestImage" 

51 sc = StorageClass(className, pytype=dict) 

52 self.assertIsInstance(sc, StorageClass) 

53 self.assertEqual(sc.name, className) 

54 self.assertEqual(str(sc), className) 

55 self.assertFalse(sc.components) 

56 self.assertTrue(sc.validateInstance({})) 

57 self.assertFalse(sc.validateInstance("")) 

58 

59 r = repr(sc) 

60 self.assertIn("StorageClass", r) 

61 self.assertIn(className, r) 

62 self.assertNotIn("parameters", r) 

63 self.assertIn("pytype='dict'", r) 

64 

65 # Ensure we do not have a delegate 

66 with self.assertRaises(TypeError): 

67 sc.delegate() 

68 

69 # Allow no definition of python type 

70 scn = StorageClass(className) 

71 self.assertIs(scn.pytype, object) 

72 

73 # Include some components 

74 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

75 self.assertIn("comp1", scc.components) 

76 r = repr(scc) 

77 self.assertIn("comp1", r) 

78 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

79 

80 # Ensure that we have a delegate 

81 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

82 

83 # Check we can create a storageClass using the name of an importable 

84 # type. 

85 sc2 = StorageClass("TestImage2", 

86 "lsst.daf.butler.core.storageClass.StorageClassFactory") 

87 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

88 self.assertIn("butler.core", repr(sc2)) 

89 

90 def testParameters(self): 

91 """Test that we can set parameters and validate them""" 

92 pt = ("a", "b") 

93 ps = {"a", "b"} 

94 pl = ["a", "b"] 

95 for p in (pt, ps, pl): 

96 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

97 self.assertEqual(sc1.parameters, ps) 

98 sc1.validateParameters(p) 

99 

100 sc1.validateParameters() 

101 sc1.validateParameters({"a": None, "b": None}) 

102 sc1.validateParameters(["a", ]) 

103 with self.assertRaises(KeyError): 

104 sc1.validateParameters({"a", "c"}) 

105 

106 def testEquality(self): 

107 """Test that StorageClass equality works""" 

108 className = "TestImage" 

109 sc1 = StorageClass(className, pytype=dict) 

110 sc2 = StorageClass(className, pytype=dict) 

111 self.assertEqual(sc1, sc2) 

112 sc3 = StorageClass(className + "2", pytype=str) 

113 self.assertNotEqual(sc1, sc3) 

114 

115 # Same StorageClass name but different python type 

116 sc4 = StorageClass(className, pytype=str) 

117 self.assertNotEqual(sc1, sc4) 

118 

119 # Parameters 

120 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

121 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

122 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

123 self.assertEqual(scp, scp1) 

124 self.assertNotEqual(scp, scp2) 

125 

126 # Now with components 

127 sc5 = StorageClass("Composite", pytype=PythonType, 

128 components={"comp1": sc1, "comp2": sc3}) 

129 sc6 = StorageClass("Composite", pytype=PythonType, 

130 components={"comp1": sc1, "comp2": sc3}) 

131 self.assertEqual(sc5, sc6) 

132 self.assertNotEqual(sc5, sc3) 

133 sc7 = StorageClass("Composite", pytype=PythonType, 

134 components={"comp1": sc4, "comp2": sc3}) 

135 self.assertNotEqual(sc5, sc7) 

136 sc8 = StorageClass("Composite", pytype=PythonType, 

137 components={"comp2": sc3, "comp3": sc3}) 

138 self.assertNotEqual(sc5, sc8) 

139 sc9 = StorageClass("Composite", pytype=PythonType, 

140 components={"comp1": sc1, "comp2": sc3}, 

141 delegate="lsst.daf.butler.Butler") 

142 self.assertNotEqual(sc5, sc9) 

143 

144 def testRegistry(self): 

145 """Check that storage classes can be created on the fly and stored 

146 in a registry.""" 

147 className = "TestImage" 

148 factory = StorageClassFactory() 

149 newclass = StorageClass(className, pytype=PythonType) 

150 factory.registerStorageClass(newclass) 

151 sc = factory.getStorageClass(className) 

152 self.assertIsInstance(sc, StorageClass) 

153 self.assertEqual(sc.name, className) 

154 self.assertFalse(sc.components) 

155 self.assertEqual(sc.pytype, PythonType) 

156 self.assertIn(sc, factory) 

157 newclass2 = StorageClass("Temporary2", pytype=str) 

158 self.assertNotIn(newclass2, factory) 

159 factory.registerStorageClass(newclass2) 

160 self.assertIn(newclass2, factory) 

161 self.assertIn("Temporary2", factory) 

162 self.assertNotIn("Temporary3", factory) 

163 self.assertNotIn({}, factory) 

164 

165 # Make sure we can't register a storage class with the same name 

166 # but different values 

167 newclass3 = StorageClass("Temporary2", pytype=dict) 

168 with self.assertRaises(ValueError): 

169 factory.registerStorageClass(newclass3) 

170 

171 factory._unregisterStorageClass(newclass3.name) 

172 self.assertNotIn(newclass3, factory) 

173 self.assertNotIn(newclass3.name, factory) 

174 factory.registerStorageClass(newclass3) 

175 self.assertIn(newclass3, factory) 

176 self.assertIn(newclass3.name, factory) 

177 

178 # Check you can silently insert something that is already there 

179 factory.registerStorageClass(newclass3) 

180 

181 def testFactoryConfig(self): 

182 factory = StorageClassFactory() 

183 factory.addFromConfig(StorageClassConfig()) 

184 image = factory.getStorageClass("Image") 

185 imageF = factory.getStorageClass("ImageF") 

186 self.assertIsInstance(imageF, type(image)) 

187 self.assertNotEqual(imageF, image) 

188 

189 # Check component inheritance 

190 exposure = factory.getStorageClass("Exposure") 

191 exposureF = factory.getStorageClass("ExposureF") 

192 self.assertIsInstance(exposureF, type(exposure)) 

193 self.assertIsInstance(exposure.components["image"], type(image)) 

194 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

195 self.assertIsInstance(exposureF.components["image"], type(image)) 

196 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

197 self.assertIn("wcs", exposure.components) 

198 self.assertIn("wcs", exposureF.components) 

199 

200 # Check parameters 

201 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

202 thing1 = factory.getStorageClass("ThingOne") 

203 thing2 = factory.getStorageClass("ThingTwo") 

204 self.assertIsInstance(thing2, type(thing1)) 

205 param1 = thing1.parameters 

206 param2 = thing2.parameters 

207 self.assertIn("param3", thing2.parameters) 

208 self.assertNotIn("param3", thing1.parameters) 

209 param2.remove("param3") 

210 self.assertEqual(param1, param2) 

211 

212 # Check that we can't have a new StorageClass that does not 

213 # inherit from StorageClass 

214 with self.assertRaises(ValueError): 

215 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

216 

217 sc = factory.makeNewStorageClass("ClassName") 

218 self.assertIsInstance(sc(), StorageClass) 

219 

220 def testPickle(self): 

221 """Test that we can pickle storageclasses. 

222 """ 

223 className = "TestImage" 

224 sc = StorageClass(className, pytype=dict) 

225 self.assertIsInstance(sc, StorageClass) 

226 self.assertEqual(sc.name, className) 

227 self.assertFalse(sc.components) 

228 sc2 = pickle.loads(pickle.dumps(sc)) 

229 self.assertEqual(sc2, sc) 

230 

231 @classmethod 

232 def _convert_type(cls, data): 

233 # Test helper function. Fail if the list is empty. 

234 if not len(data): 

235 raise RuntimeError("Deliberate failure.") 

236 return {"key": data} 

237 

238 def testConverters(self): 

239 """Test conversion maps.""" 

240 

241 className = "TestConverters" 

242 converters = { 

243 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

244 # Add some entries that will fail to import. 

245 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

246 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

247 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

248 "list": get_full_type_name(self._convert_type), 

249 } 

250 sc = StorageClass(className, pytype=dict, converters=converters) 

251 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

252 sc2 = StorageClass("Test2", pytype=set) 

253 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

254 

255 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

256 # Initially the converter list is not filtered. 

257 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

258 self.assertNotIn("converters", repr(sc2)) 

259 

260 self.assertTrue(sc.can_convert(sc)) 

261 self.assertFalse(sc.can_convert(sc2)) 

262 self.assertTrue(sc.can_convert(sc3)) 

263 

264 # After we've processed the converters the bad ones will no longer 

265 # be reported. 

266 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

267 self.assertEqual(len(sc.converters), 2) 

268 

269 self.assertIsNone(sc.coerce_type(None)) 

270 

271 converted = sc.coerce_type([1, 2, 3]) 

272 self.assertEqual(converted, {"key": [1, 2, 3]}) 

273 

274 # Try to coerce a type that is not supported. 

275 with self.assertRaises(TypeError): 

276 sc.coerce_type(set([1, 2, 3])) 

277 

278 # Coerce something that will fail to convert. 

279 with self.assertLogs(level=logging.ERROR) as cm: 

280 with self.assertRaises(RuntimeError): 

281 sc.coerce_type([]) 

282 self.assertIn("failed to convert type list", cm.output[0]) 

283 

284 

285if __name__ == "__main__": 285 ↛ 286line 285 didn't jump to line 286, because the condition on line 285 was never true

286 unittest.main()