Coverage for tests/test_storageClass.py: 12%

199 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-03 02:30 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import pickle 

25import unittest 

26 

27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

28from lsst.daf.butler.tests import MetricsExample 

29from lsst.utils.introspection import get_full_type_name 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 

40 pass 

41 

42 

43class StorageClassFactoryTestCase(unittest.TestCase): 

44 """Tests of the storage class infrastructure.""" 

45 

46 def testCreation(self): 

47 """Test that we can dynamically create storage class subclasses. 

48 

49 This is critical for testing the factory functions.""" 

50 className = "TestImage" 

51 sc = StorageClass(className, pytype=dict) 

52 self.assertIsInstance(sc, StorageClass) 

53 self.assertEqual(sc.name, className) 

54 self.assertEqual(str(sc), className) 

55 self.assertFalse(sc.components) 

56 self.assertTrue(sc.validateInstance({})) 

57 self.assertFalse(sc.validateInstance("")) 

58 

59 r = repr(sc) 

60 self.assertIn("StorageClass", r) 

61 self.assertIn(className, r) 

62 self.assertNotIn("parameters", r) 

63 self.assertIn("pytype='dict'", r) 

64 

65 # Ensure we do not have a delegate 

66 with self.assertRaises(TypeError): 

67 sc.delegate() 

68 

69 # Allow no definition of python type 

70 scn = StorageClass(className) 

71 self.assertIs(scn.pytype, object) 

72 

73 # Include some components 

74 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

75 self.assertIn("comp1", scc.components) 

76 r = repr(scc) 

77 self.assertIn("comp1", r) 

78 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

79 

80 # Ensure that we have a delegate 

81 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

82 

83 # Check we can create a storageClass using the name of an importable 

84 # type. 

85 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

86 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

87 self.assertIn("butler.core", repr(sc2)) 

88 

89 def testParameters(self): 

90 """Test that we can set parameters and validate them""" 

91 pt = ("a", "b") 

92 ps = {"a", "b"} 

93 pl = ["a", "b"] 

94 for p in (pt, ps, pl): 

95 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

96 self.assertEqual(sc1.parameters, ps) 

97 sc1.validateParameters(p) 

98 

99 sc1.validateParameters() 

100 sc1.validateParameters({"a": None, "b": None}) 

101 sc1.validateParameters( 

102 [ 

103 "a", 

104 ] 

105 ) 

106 with self.assertRaises(KeyError): 

107 sc1.validateParameters({"a", "c"}) 

108 

109 def testEquality(self): 

110 """Test that StorageClass equality works""" 

111 className = "TestImage" 

112 sc1 = StorageClass(className, pytype=dict) 

113 sc2 = StorageClass(className, pytype=dict) 

114 self.assertEqual(sc1, sc2) 

115 sc3 = StorageClass(className + "2", pytype=str) 

116 self.assertNotEqual(sc1, sc3) 

117 

118 # Same StorageClass name but different python type 

119 sc4 = StorageClass(className, pytype=str) 

120 self.assertNotEqual(sc1, sc4) 

121 

122 # Parameters 

123 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

124 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

125 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

126 self.assertEqual(scp, scp1) 

127 self.assertNotEqual(scp, scp2) 

128 

129 # Now with components 

130 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

131 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

132 self.assertEqual(sc5, sc6) 

133 self.assertNotEqual(sc5, sc3) 

134 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

135 self.assertNotEqual(sc5, sc7) 

136 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

137 self.assertNotEqual(sc5, sc8) 

138 sc9 = StorageClass( 

139 "Composite", 

140 pytype=PythonType, 

141 components={"comp1": sc1, "comp2": sc3}, 

142 delegate="lsst.daf.butler.Butler", 

143 ) 

144 self.assertNotEqual(sc5, sc9) 

145 

146 def testTypeEquality(self): 

147 sc1 = StorageClass("Something", pytype=dict) 

148 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

149 self.assertFalse(sc1.is_type(str), repr(sc1)) 

150 

151 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

152 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

153 

154 def testRegistry(self): 

155 """Check that storage classes can be created on the fly and stored 

156 in a registry.""" 

157 className = "TestImage" 

158 factory = StorageClassFactory() 

159 newclass = StorageClass(className, pytype=PythonType) 

160 factory.registerStorageClass(newclass) 

161 sc = factory.getStorageClass(className) 

162 self.assertIsInstance(sc, StorageClass) 

163 self.assertEqual(sc.name, className) 

164 self.assertFalse(sc.components) 

165 self.assertEqual(sc.pytype, PythonType) 

166 self.assertIn(sc, factory) 

167 newclass2 = StorageClass("Temporary2", pytype=str) 

168 self.assertNotIn(newclass2, factory) 

169 factory.registerStorageClass(newclass2) 

170 self.assertIn(newclass2, factory) 

171 self.assertIn("Temporary2", factory) 

172 self.assertNotIn("Temporary3", factory) 

173 self.assertNotIn({}, factory) 

174 

175 # Make sure iterators work. 

176 keys = set(factory.keys()) 

177 self.assertIn("Temporary2", keys) 

178 

179 iterkeys = {k for k in factory} 

180 self.assertEqual(keys, iterkeys) 

181 

182 values = set(factory.values()) 

183 self.assertIn(sc, values) 

184 self.assertEqual(len(factory), len(values)) 

185 

186 external = {k: v for k, v in factory.items()} 

187 self.assertIn("Temporary2", external) 

188 

189 # Make sure we can't register a storage class with the same name 

190 # but different values 

191 newclass3 = StorageClass("Temporary2", pytype=dict) 

192 with self.assertRaises(ValueError) as cm: 

193 factory.registerStorageClass(newclass3) 

194 self.assertIn("pytype='dict'", str(cm.exception)) 

195 with self.assertRaises(ValueError) as cm: 

196 factory.registerStorageClass(newclass3, msg="error string") 

197 self.assertIn("error string", str(cm.exception)) 

198 

199 factory._unregisterStorageClass(newclass3.name) 

200 self.assertNotIn(newclass3, factory) 

201 self.assertNotIn(newclass3.name, factory) 

202 factory.registerStorageClass(newclass3) 

203 self.assertIn(newclass3, factory) 

204 self.assertIn(newclass3.name, factory) 

205 

206 # Check you can silently insert something that is already there 

207 factory.registerStorageClass(newclass3) 

208 

209 def testFactoryConfig(self): 

210 factory = StorageClassFactory() 

211 factory.addFromConfig(StorageClassConfig()) 

212 image = factory.getStorageClass("Image") 

213 imageF = factory.getStorageClass("ImageF") 

214 self.assertIsInstance(imageF, type(image)) 

215 self.assertNotEqual(imageF, image) 

216 

217 # Check component inheritance 

218 exposure = factory.getStorageClass("Exposure") 

219 exposureF = factory.getStorageClass("ExposureF") 

220 self.assertIsInstance(exposureF, type(exposure)) 

221 self.assertIsInstance(exposure.components["image"], type(image)) 

222 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

223 self.assertIsInstance(exposureF.components["image"], type(image)) 

224 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

225 self.assertIn("wcs", exposure.components) 

226 self.assertIn("wcs", exposureF.components) 

227 

228 # Check parameters 

229 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

230 thing1 = factory.getStorageClass("ThingOne") 

231 thing2 = factory.getStorageClass("ThingTwo") 

232 self.assertIsInstance(thing2, type(thing1)) 

233 param1 = thing1.parameters 

234 param2 = thing2.parameters 

235 self.assertIn("param3", thing2.parameters) 

236 self.assertNotIn("param3", thing1.parameters) 

237 param2.remove("param3") 

238 self.assertEqual(param1, param2) 

239 

240 # Check that we can't have a new StorageClass that does not 

241 # inherit from StorageClass 

242 with self.assertRaises(ValueError): 

243 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

244 

245 sc = factory.makeNewStorageClass("ClassName") 

246 self.assertIsInstance(sc(), StorageClass) 

247 

248 def testPickle(self): 

249 """Test that we can pickle storageclasses.""" 

250 className = "TestImage" 

251 sc = StorageClass(className, pytype=dict) 

252 self.assertIsInstance(sc, StorageClass) 

253 self.assertEqual(sc.name, className) 

254 self.assertFalse(sc.components) 

255 sc2 = pickle.loads(pickle.dumps(sc)) 

256 self.assertEqual(sc2, sc) 

257 

258 @classmethod 

259 def _convert_type(cls, data): 

260 # Test helper function. Fail if the list is empty. 

261 if not len(data): 

262 raise RuntimeError("Deliberate failure.") 

263 return {"key": data} 

264 

265 def testConverters(self): 

266 """Test conversion maps.""" 

267 

268 className = "TestConverters" 

269 converters = { 

270 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

271 # Add some entries that will fail to import. 

272 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

273 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

274 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

275 "list": get_full_type_name(self._convert_type), 

276 } 

277 sc = StorageClass(className, pytype=dict, converters=converters) 

278 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

279 sc2 = StorageClass("Test2", pytype=set) 

280 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

281 

282 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

283 # Initially the converter list is not filtered. 

284 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

285 self.assertNotIn("converters", repr(sc2)) 

286 

287 self.assertTrue(sc.can_convert(sc)) 

288 self.assertFalse(sc.can_convert(sc2)) 

289 self.assertTrue(sc.can_convert(sc3)) 

290 

291 # After we've processed the converters the bad ones will no longer 

292 # be reported. 

293 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

294 self.assertEqual(len(sc.converters), 2) 

295 

296 self.assertIsNone(sc.coerce_type(None)) 

297 

298 converted = sc.coerce_type([1, 2, 3]) 

299 self.assertEqual(converted, {"key": [1, 2, 3]}) 

300 

301 # Convert Metrics using a named method converter. 

302 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

303 converted = sc.coerce_type(metric) 

304 self.assertEqual(converted["data"], [1, 2], converted) 

305 

306 # Check that python types matching is allowed. 

307 sc4 = StorageClass("Test2", pytype=set) 

308 self.assertTrue(sc2.can_convert(sc4)) 

309 converted = sc2.coerce_type({1, 2}) 

310 self.assertEqual(converted, {1, 2}) 

311 

312 # Try to coerce a type that is not supported. 

313 with self.assertRaises(TypeError): 

314 sc.coerce_type(set([1, 2, 3])) 

315 

316 # Coerce something that will fail to convert. 

317 with self.assertLogs(level=logging.ERROR) as cm: 

318 with self.assertRaises(RuntimeError): 

319 sc.coerce_type([]) 

320 self.assertIn("failed to convert type list", cm.output[0]) 

321 

322 

323if __name__ == "__main__": 323 ↛ 324line 323 didn't jump to line 324, because the condition on line 323 was never true

324 unittest.main()