Coverage for tests/test_storageClass.py: 13%

184 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-21 02:43 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import pickle 

25import unittest 

26 

27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

28from lsst.daf.butler.tests import MetricsExample 

29from lsst.utils.introspection import get_full_type_name 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 

40 pass 

41 

42 

43class StorageClassFactoryTestCase(unittest.TestCase): 

44 """Tests of the storage class infrastructure.""" 

45 

46 def testCreation(self): 

47 """Test that we can dynamically create storage class subclasses. 

48 

49 This is critical for testing the factory functions.""" 

50 className = "TestImage" 

51 sc = StorageClass(className, pytype=dict) 

52 self.assertIsInstance(sc, StorageClass) 

53 self.assertEqual(sc.name, className) 

54 self.assertEqual(str(sc), className) 

55 self.assertFalse(sc.components) 

56 self.assertTrue(sc.validateInstance({})) 

57 self.assertFalse(sc.validateInstance("")) 

58 

59 r = repr(sc) 

60 self.assertIn("StorageClass", r) 

61 self.assertIn(className, r) 

62 self.assertNotIn("parameters", r) 

63 self.assertIn("pytype='dict'", r) 

64 

65 # Ensure we do not have a delegate 

66 with self.assertRaises(TypeError): 

67 sc.delegate() 

68 

69 # Allow no definition of python type 

70 scn = StorageClass(className) 

71 self.assertIs(scn.pytype, object) 

72 

73 # Include some components 

74 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

75 self.assertIn("comp1", scc.components) 

76 r = repr(scc) 

77 self.assertIn("comp1", r) 

78 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

79 

80 # Ensure that we have a delegate 

81 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

82 

83 # Check we can create a storageClass using the name of an importable 

84 # type. 

85 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

86 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

87 self.assertIn("butler.core", repr(sc2)) 

88 

89 def testParameters(self): 

90 """Test that we can set parameters and validate them""" 

91 pt = ("a", "b") 

92 ps = {"a", "b"} 

93 pl = ["a", "b"] 

94 for p in (pt, ps, pl): 

95 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

96 self.assertEqual(sc1.parameters, ps) 

97 sc1.validateParameters(p) 

98 

99 sc1.validateParameters() 

100 sc1.validateParameters({"a": None, "b": None}) 

101 sc1.validateParameters( 

102 [ 

103 "a", 

104 ] 

105 ) 

106 with self.assertRaises(KeyError): 

107 sc1.validateParameters({"a", "c"}) 

108 

109 def testEquality(self): 

110 """Test that StorageClass equality works""" 

111 className = "TestImage" 

112 sc1 = StorageClass(className, pytype=dict) 

113 sc2 = StorageClass(className, pytype=dict) 

114 self.assertEqual(sc1, sc2) 

115 sc3 = StorageClass(className + "2", pytype=str) 

116 self.assertNotEqual(sc1, sc3) 

117 

118 # Same StorageClass name but different python type 

119 sc4 = StorageClass(className, pytype=str) 

120 self.assertNotEqual(sc1, sc4) 

121 

122 # Parameters 

123 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

124 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

125 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

126 self.assertEqual(scp, scp1) 

127 self.assertNotEqual(scp, scp2) 

128 

129 # Now with components 

130 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

131 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

132 self.assertEqual(sc5, sc6) 

133 self.assertNotEqual(sc5, sc3) 

134 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

135 self.assertNotEqual(sc5, sc7) 

136 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

137 self.assertNotEqual(sc5, sc8) 

138 sc9 = StorageClass( 

139 "Composite", 

140 pytype=PythonType, 

141 components={"comp1": sc1, "comp2": sc3}, 

142 delegate="lsst.daf.butler.Butler", 

143 ) 

144 self.assertNotEqual(sc5, sc9) 

145 

146 def testRegistry(self): 

147 """Check that storage classes can be created on the fly and stored 

148 in a registry.""" 

149 className = "TestImage" 

150 factory = StorageClassFactory() 

151 newclass = StorageClass(className, pytype=PythonType) 

152 factory.registerStorageClass(newclass) 

153 sc = factory.getStorageClass(className) 

154 self.assertIsInstance(sc, StorageClass) 

155 self.assertEqual(sc.name, className) 

156 self.assertFalse(sc.components) 

157 self.assertEqual(sc.pytype, PythonType) 

158 self.assertIn(sc, factory) 

159 newclass2 = StorageClass("Temporary2", pytype=str) 

160 self.assertNotIn(newclass2, factory) 

161 factory.registerStorageClass(newclass2) 

162 self.assertIn(newclass2, factory) 

163 self.assertIn("Temporary2", factory) 

164 self.assertNotIn("Temporary3", factory) 

165 self.assertNotIn({}, factory) 

166 

167 # Make sure we can't register a storage class with the same name 

168 # but different values 

169 newclass3 = StorageClass("Temporary2", pytype=dict) 

170 with self.assertRaises(ValueError) as cm: 

171 factory.registerStorageClass(newclass3) 

172 self.assertIn("pytype='dict'", str(cm.exception)) 

173 with self.assertRaises(ValueError) as cm: 

174 factory.registerStorageClass(newclass3, msg="error string") 

175 self.assertIn("error string", str(cm.exception)) 

176 

177 factory._unregisterStorageClass(newclass3.name) 

178 self.assertNotIn(newclass3, factory) 

179 self.assertNotIn(newclass3.name, factory) 

180 factory.registerStorageClass(newclass3) 

181 self.assertIn(newclass3, factory) 

182 self.assertIn(newclass3.name, factory) 

183 

184 # Check you can silently insert something that is already there 

185 factory.registerStorageClass(newclass3) 

186 

187 def testFactoryConfig(self): 

188 factory = StorageClassFactory() 

189 factory.addFromConfig(StorageClassConfig()) 

190 image = factory.getStorageClass("Image") 

191 imageF = factory.getStorageClass("ImageF") 

192 self.assertIsInstance(imageF, type(image)) 

193 self.assertNotEqual(imageF, image) 

194 

195 # Check component inheritance 

196 exposure = factory.getStorageClass("Exposure") 

197 exposureF = factory.getStorageClass("ExposureF") 

198 self.assertIsInstance(exposureF, type(exposure)) 

199 self.assertIsInstance(exposure.components["image"], type(image)) 

200 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

201 self.assertIsInstance(exposureF.components["image"], type(image)) 

202 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

203 self.assertIn("wcs", exposure.components) 

204 self.assertIn("wcs", exposureF.components) 

205 

206 # Check parameters 

207 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

208 thing1 = factory.getStorageClass("ThingOne") 

209 thing2 = factory.getStorageClass("ThingTwo") 

210 self.assertIsInstance(thing2, type(thing1)) 

211 param1 = thing1.parameters 

212 param2 = thing2.parameters 

213 self.assertIn("param3", thing2.parameters) 

214 self.assertNotIn("param3", thing1.parameters) 

215 param2.remove("param3") 

216 self.assertEqual(param1, param2) 

217 

218 # Check that we can't have a new StorageClass that does not 

219 # inherit from StorageClass 

220 with self.assertRaises(ValueError): 

221 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

222 

223 sc = factory.makeNewStorageClass("ClassName") 

224 self.assertIsInstance(sc(), StorageClass) 

225 

226 def testPickle(self): 

227 """Test that we can pickle storageclasses.""" 

228 className = "TestImage" 

229 sc = StorageClass(className, pytype=dict) 

230 self.assertIsInstance(sc, StorageClass) 

231 self.assertEqual(sc.name, className) 

232 self.assertFalse(sc.components) 

233 sc2 = pickle.loads(pickle.dumps(sc)) 

234 self.assertEqual(sc2, sc) 

235 

236 @classmethod 

237 def _convert_type(cls, data): 

238 # Test helper function. Fail if the list is empty. 

239 if not len(data): 

240 raise RuntimeError("Deliberate failure.") 

241 return {"key": data} 

242 

243 def testConverters(self): 

244 """Test conversion maps.""" 

245 

246 className = "TestConverters" 

247 converters = { 

248 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

249 # Add some entries that will fail to import. 

250 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

251 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

252 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

253 "list": get_full_type_name(self._convert_type), 

254 } 

255 sc = StorageClass(className, pytype=dict, converters=converters) 

256 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

257 sc2 = StorageClass("Test2", pytype=set) 

258 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

259 

260 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

261 # Initially the converter list is not filtered. 

262 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

263 self.assertNotIn("converters", repr(sc2)) 

264 

265 self.assertTrue(sc.can_convert(sc)) 

266 self.assertFalse(sc.can_convert(sc2)) 

267 self.assertTrue(sc.can_convert(sc3)) 

268 

269 # After we've processed the converters the bad ones will no longer 

270 # be reported. 

271 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

272 self.assertEqual(len(sc.converters), 2) 

273 

274 self.assertIsNone(sc.coerce_type(None)) 

275 

276 converted = sc.coerce_type([1, 2, 3]) 

277 self.assertEqual(converted, {"key": [1, 2, 3]}) 

278 

279 # Convert Metrics using a named method converter. 

280 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

281 converted = sc.coerce_type(metric) 

282 self.assertEqual(converted["data"], [1, 2], converted) 

283 

284 # Check that python types matching is allowed. 

285 sc4 = StorageClass("Test2", pytype=set) 

286 self.assertTrue(sc2.can_convert(sc4)) 

287 converted = sc2.coerce_type({1, 2}) 

288 self.assertEqual(converted, {1, 2}) 

289 

290 # Try to coerce a type that is not supported. 

291 with self.assertRaises(TypeError): 

292 sc.coerce_type(set([1, 2, 3])) 

293 

294 # Coerce something that will fail to convert. 

295 with self.assertLogs(level=logging.ERROR) as cm: 

296 with self.assertRaises(RuntimeError): 

297 sc.coerce_type([]) 

298 self.assertIn("failed to convert type list", cm.output[0]) 

299 

300 

301if __name__ == "__main__": 301 ↛ 302line 301 didn't jump to line 302, because the condition on line 301 was never true

302 unittest.main()