Coverage for tests/test_storageClass.py: 12%

216 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-12 02:05 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import pickle 

25import unittest 

26 

27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

28from lsst.daf.butler.tests import MetricsExample 

29from lsst.utils.introspection import get_full_type_name 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 

40 pass 

41 

42 

43class PythonType2: 

44 """A dummy class to test the registry of Python types.""" 

45 

46 pass 

47 

48 

49class PythonType3: 

50 """A dummy class to test the registry of Python types.""" 

51 

52 pass 

53 

54 

55class StorageClassFactoryTestCase(unittest.TestCase): 

56 """Tests of the storage class infrastructure.""" 

57 

58 def testCreation(self): 

59 """Test that we can dynamically create storage class subclasses. 

60 

61 This is critical for testing the factory functions.""" 

62 className = "TestImage" 

63 sc = StorageClass(className, pytype=dict) 

64 self.assertIsInstance(sc, StorageClass) 

65 self.assertEqual(sc.name, className) 

66 self.assertEqual(str(sc), className) 

67 self.assertFalse(sc.components) 

68 self.assertTrue(sc.validateInstance({})) 

69 self.assertFalse(sc.validateInstance("")) 

70 

71 r = repr(sc) 

72 self.assertIn("StorageClass", r) 

73 self.assertIn(className, r) 

74 self.assertNotIn("parameters", r) 

75 self.assertIn("pytype='dict'", r) 

76 

77 # Ensure we do not have a delegate 

78 with self.assertRaises(TypeError): 

79 sc.delegate() 

80 

81 # Allow no definition of python type 

82 scn = StorageClass(className) 

83 self.assertIs(scn.pytype, object) 

84 

85 # Include some components 

86 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

87 self.assertIn("comp1", scc.components) 

88 r = repr(scc) 

89 self.assertIn("comp1", r) 

90 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

91 

92 # Ensure that we have a delegate 

93 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

94 

95 # Check we can create a storageClass using the name of an importable 

96 # type. 

97 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

98 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

99 self.assertIn("butler.core", repr(sc2)) 

100 

101 def testParameters(self): 

102 """Test that we can set parameters and validate them""" 

103 pt = ("a", "b") 

104 ps = {"a", "b"} 

105 pl = ["a", "b"] 

106 for p in (pt, ps, pl): 

107 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

108 self.assertEqual(sc1.parameters, ps) 

109 sc1.validateParameters(p) 

110 

111 sc1.validateParameters() 

112 sc1.validateParameters({"a": None, "b": None}) 

113 sc1.validateParameters( 

114 [ 

115 "a", 

116 ] 

117 ) 

118 with self.assertRaises(KeyError): 

119 sc1.validateParameters({"a", "c"}) 

120 

121 def testEquality(self): 

122 """Test that StorageClass equality works""" 

123 className = "TestImage" 

124 sc1 = StorageClass(className, pytype=dict) 

125 sc2 = StorageClass(className, pytype=dict) 

126 self.assertEqual(sc1, sc2) 

127 sc3 = StorageClass(className + "2", pytype=str) 

128 self.assertNotEqual(sc1, sc3) 

129 

130 # Same StorageClass name but different python type 

131 sc4 = StorageClass(className, pytype=str) 

132 self.assertNotEqual(sc1, sc4) 

133 

134 # Parameters 

135 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

136 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

137 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

138 self.assertEqual(scp, scp1) 

139 self.assertNotEqual(scp, scp2) 

140 

141 # Now with components 

142 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

143 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

144 self.assertEqual(sc5, sc6) 

145 self.assertNotEqual(sc5, sc3) 

146 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

147 self.assertNotEqual(sc5, sc7) 

148 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

149 self.assertNotEqual(sc5, sc8) 

150 sc9 = StorageClass( 

151 "Composite", 

152 pytype=PythonType, 

153 components={"comp1": sc1, "comp2": sc3}, 

154 delegate="lsst.daf.butler.Butler", 

155 ) 

156 self.assertNotEqual(sc5, sc9) 

157 

158 def testTypeEquality(self): 

159 sc1 = StorageClass("Something", pytype=dict) 

160 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

161 self.assertFalse(sc1.is_type(str), repr(sc1)) 

162 

163 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

164 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

165 

166 def testRegistry(self): 

167 """Check that storage classes can be created on the fly and stored 

168 in a registry.""" 

169 className = "TestImage" 

170 factory = StorageClassFactory() 

171 newclass = StorageClass(className, pytype=PythonType) 

172 factory.registerStorageClass(newclass) 

173 sc = factory.getStorageClass(className) 

174 self.assertIsInstance(sc, StorageClass) 

175 self.assertEqual(sc.name, className) 

176 self.assertFalse(sc.components) 

177 self.assertEqual(sc.pytype, PythonType) 

178 self.assertIn(sc, factory) 

179 newclass2 = StorageClass("Temporary2", pytype=str) 

180 self.assertNotIn(newclass2, factory) 

181 factory.registerStorageClass(newclass2) 

182 self.assertIn(newclass2, factory) 

183 self.assertIn("Temporary2", factory) 

184 self.assertNotIn("Temporary3", factory) 

185 self.assertNotIn({}, factory) 

186 

187 # Make sure iterators work. 

188 keys = set(factory.keys()) 

189 self.assertIn("Temporary2", keys) 

190 

191 iterkeys = {k for k in factory} 

192 self.assertEqual(keys, iterkeys) 

193 

194 values = set(factory.values()) 

195 self.assertIn(sc, values) 

196 self.assertEqual(len(factory), len(values)) 

197 

198 external = {k: v for k, v in factory.items()} 

199 self.assertIn("Temporary2", external) 

200 

201 # Make sure we can't register a storage class with the same name 

202 # but different values 

203 newclass3 = StorageClass("Temporary2", pytype=dict) 

204 with self.assertRaises(ValueError) as cm: 

205 factory.registerStorageClass(newclass3) 

206 self.assertIn("pytype='dict'", str(cm.exception)) 

207 with self.assertRaises(ValueError) as cm: 

208 factory.registerStorageClass(newclass3, msg="error string") 

209 self.assertIn("error string", str(cm.exception)) 

210 

211 factory._unregisterStorageClass(newclass3.name) 

212 self.assertNotIn(newclass3, factory) 

213 self.assertNotIn(newclass3.name, factory) 

214 factory.registerStorageClass(newclass3) 

215 self.assertIn(newclass3, factory) 

216 self.assertIn(newclass3.name, factory) 

217 

218 # Check you can silently insert something that is already there 

219 factory.registerStorageClass(newclass3) 

220 

221 def testFactoryFind(self): 

222 # Finding a storage class can involve doing lots of slow imports so 

223 # this is a separate test. 

224 factory = StorageClassFactory() 

225 className = "PythonType3" 

226 newclass = StorageClass(className, pytype=PythonType3) 

227 factory.registerStorageClass(newclass) 

228 sc = factory.getStorageClass(className) 

229 

230 # Can we find a storage class from a type. 

231 new_sc = factory.findStorageClass(PythonType3) 

232 self.assertEqual(new_sc, sc) 

233 

234 # Now with slow mode 

235 new_sc = factory.findStorageClass(PythonType3, compare_types=True) 

236 self.assertEqual(new_sc, sc) 

237 

238 # This class will never match. 

239 with self.assertRaises(KeyError): 

240 factory.findStorageClass(PythonType2, compare_types=True) 

241 

242 # Check builtins. 

243 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict")) 

244 

245 def testFactoryConfig(self): 

246 factory = StorageClassFactory() 

247 factory.addFromConfig(StorageClassConfig()) 

248 image = factory.getStorageClass("Image") 

249 imageF = factory.getStorageClass("ImageF") 

250 self.assertIsInstance(imageF, type(image)) 

251 self.assertNotEqual(imageF, image) 

252 

253 # Check component inheritance 

254 exposure = factory.getStorageClass("Exposure") 

255 exposureF = factory.getStorageClass("ExposureF") 

256 self.assertIsInstance(exposureF, type(exposure)) 

257 self.assertIsInstance(exposure.components["image"], type(image)) 

258 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

259 self.assertIsInstance(exposureF.components["image"], type(image)) 

260 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

261 self.assertIn("wcs", exposure.components) 

262 self.assertIn("wcs", exposureF.components) 

263 

264 # Check parameters 

265 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

266 thing1 = factory.getStorageClass("ThingOne") 

267 thing2 = factory.getStorageClass("ThingTwo") 

268 self.assertIsInstance(thing2, type(thing1)) 

269 param1 = thing1.parameters 

270 param2 = thing2.parameters 

271 self.assertIn("param3", thing2.parameters) 

272 self.assertNotIn("param3", thing1.parameters) 

273 param2.remove("param3") 

274 self.assertEqual(param1, param2) 

275 

276 # Check that we can't have a new StorageClass that does not 

277 # inherit from StorageClass 

278 with self.assertRaises(ValueError): 

279 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

280 

281 sc = factory.makeNewStorageClass("ClassName") 

282 self.assertIsInstance(sc(), StorageClass) 

283 

284 def testPickle(self): 

285 """Test that we can pickle storageclasses.""" 

286 className = "TestImage" 

287 sc = StorageClass(className, pytype=dict) 

288 self.assertIsInstance(sc, StorageClass) 

289 self.assertEqual(sc.name, className) 

290 self.assertFalse(sc.components) 

291 sc2 = pickle.loads(pickle.dumps(sc)) 

292 self.assertEqual(sc2, sc) 

293 

294 @classmethod 

295 def _convert_type(cls, data): 

296 # Test helper function. Fail if the list is empty. 

297 if not len(data): 

298 raise RuntimeError("Deliberate failure.") 

299 return {"key": data} 

300 

301 def testConverters(self): 

302 """Test conversion maps.""" 

303 

304 className = "TestConverters" 

305 converters = { 

306 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

307 # Add some entries that will fail to import. 

308 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

309 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

310 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

311 "list": get_full_type_name(self._convert_type), 

312 } 

313 sc = StorageClass(className, pytype=dict, converters=converters) 

314 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

315 sc2 = StorageClass("Test2", pytype=set) 

316 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

317 

318 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

319 # Initially the converter list is not filtered. 

320 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

321 self.assertNotIn("converters", repr(sc2)) 

322 

323 self.assertTrue(sc.can_convert(sc)) 

324 self.assertFalse(sc.can_convert(sc2)) 

325 self.assertTrue(sc.can_convert(sc3)) 

326 

327 # After we've processed the converters the bad ones will no longer 

328 # be reported. 

329 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

330 self.assertEqual(len(sc.converters), 2) 

331 

332 self.assertIsNone(sc.coerce_type(None)) 

333 

334 converted = sc.coerce_type([1, 2, 3]) 

335 self.assertEqual(converted, {"key": [1, 2, 3]}) 

336 

337 # Convert Metrics using a named method converter. 

338 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

339 converted = sc.coerce_type(metric) 

340 self.assertEqual(converted["data"], [1, 2], converted) 

341 

342 # Check that python types matching is allowed. 

343 sc4 = StorageClass("Test2", pytype=set) 

344 self.assertTrue(sc2.can_convert(sc4)) 

345 converted = sc2.coerce_type({1, 2}) 

346 self.assertEqual(converted, {1, 2}) 

347 

348 # Try to coerce a type that is not supported. 

349 with self.assertRaises(TypeError): 

350 sc.coerce_type(set([1, 2, 3])) 

351 

352 # Coerce something that will fail to convert. 

353 with self.assertLogs(level=logging.ERROR) as cm: 

354 with self.assertRaises(RuntimeError): 

355 sc.coerce_type([]) 

356 self.assertIn("failed to convert type list", cm.output[0]) 

357 

358 

359if __name__ == "__main__": 359 ↛ 360line 359 didn't jump to line 360, because the condition on line 359 was never true

360 unittest.main()