Coverage for tests/test_storageClass.py: 12%

224 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 09:20 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import pickle 

25import unittest 

26 

27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

28from lsst.daf.butler.tests import MetricsExample 

29from lsst.utils.introspection import get_full_type_name 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 

40 pass 

41 

42 

43class PythonType2: 

44 """A dummy class to test the registry of Python types.""" 

45 

46 pass 

47 

48 

49class PythonType3: 

50 """A dummy class to test the registry of Python types.""" 

51 

52 pass 

53 

54 

55class NotCopyable: 

56 """Class with deep copying disabled.""" 

57 

58 def __deepcopy__(self, memo=None): 

59 raise RuntimeError("Can not be copied.") 

60 

61 

62class StorageClassFactoryTestCase(unittest.TestCase): 

63 """Tests of the storage class infrastructure.""" 

64 

65 def testCreation(self): 

66 """Test that we can dynamically create storage class subclasses. 

67 

68 This is critical for testing the factory functions. 

69 """ 

70 className = "TestImage" 

71 sc = StorageClass(className, pytype=dict) 

72 self.assertIsInstance(sc, StorageClass) 

73 self.assertEqual(sc.name, className) 

74 self.assertEqual(str(sc), className) 

75 self.assertFalse(sc.components) 

76 self.assertTrue(sc.validateInstance({})) 

77 self.assertFalse(sc.validateInstance("")) 

78 

79 r = repr(sc) 

80 self.assertIn("StorageClass", r) 

81 self.assertIn(className, r) 

82 self.assertNotIn("parameters", r) 

83 self.assertIn("pytype='dict'", r) 

84 

85 # Ensure we do not have a delegate 

86 with self.assertRaises(TypeError): 

87 sc.delegate() 

88 

89 # Allow no definition of python type 

90 scn = StorageClass(className) 

91 self.assertIs(scn.pytype, object) 

92 

93 # Include some components 

94 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

95 self.assertIn("comp1", scc.components) 

96 r = repr(scc) 

97 self.assertIn("comp1", r) 

98 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

99 

100 # Ensure that we have a delegate 

101 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

102 

103 # Check that delegate copy() works. 

104 list1 = [1, 2, 3] 

105 list2 = scc.delegate().copy(list1) 

106 self.assertEqual(list1, list2) 

107 list2.append(4) 

108 self.assertNotEqual(list1, list2) 

109 

110 with self.assertRaises(NotImplementedError): 

111 scc.delegate().copy(NotCopyable()) 

112 

113 # Check we can create a storageClass using the name of an importable 

114 # type. 

115 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

116 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

117 self.assertIn("butler.core", repr(sc2)) 

118 

119 def testParameters(self): 

120 """Test that we can set parameters and validate them""" 

121 pt = ("a", "b") 

122 ps = {"a", "b"} 

123 pl = ["a", "b"] 

124 for p in (pt, ps, pl): 

125 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

126 self.assertEqual(sc1.parameters, ps) 

127 sc1.validateParameters(p) 

128 

129 sc1.validateParameters() 

130 sc1.validateParameters({"a": None, "b": None}) 

131 sc1.validateParameters( 

132 [ 

133 "a", 

134 ] 

135 ) 

136 with self.assertRaises(KeyError): 

137 sc1.validateParameters({"a", "c"}) 

138 

139 def testEquality(self): 

140 """Test that StorageClass equality works""" 

141 className = "TestImage" 

142 sc1 = StorageClass(className, pytype=dict) 

143 sc2 = StorageClass(className, pytype=dict) 

144 self.assertEqual(sc1, sc2) 

145 sc3 = StorageClass(className + "2", pytype=str) 

146 self.assertNotEqual(sc1, sc3) 

147 

148 # Same StorageClass name but different python type 

149 sc4 = StorageClass(className, pytype=str) 

150 self.assertNotEqual(sc1, sc4) 

151 

152 # Parameters 

153 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

154 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

155 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

156 self.assertEqual(scp, scp1) 

157 self.assertNotEqual(scp, scp2) 

158 

159 # Now with components 

160 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

161 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

162 self.assertEqual(sc5, sc6) 

163 self.assertNotEqual(sc5, sc3) 

164 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

165 self.assertNotEqual(sc5, sc7) 

166 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

167 self.assertNotEqual(sc5, sc8) 

168 sc9 = StorageClass( 

169 "Composite", 

170 pytype=PythonType, 

171 components={"comp1": sc1, "comp2": sc3}, 

172 delegate="lsst.daf.butler.Butler", 

173 ) 

174 self.assertNotEqual(sc5, sc9) 

175 

176 def testTypeEquality(self): 

177 sc1 = StorageClass("Something", pytype=dict) 

178 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

179 self.assertFalse(sc1.is_type(str), repr(sc1)) 

180 

181 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

182 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

183 

184 def testRegistry(self): 

185 """Check that storage classes can be created on the fly and stored 

186 in a registry. 

187 """ 

188 className = "TestImage" 

189 factory = StorageClassFactory() 

190 newclass = StorageClass(className, pytype=PythonType) 

191 factory.registerStorageClass(newclass) 

192 sc = factory.getStorageClass(className) 

193 self.assertIsInstance(sc, StorageClass) 

194 self.assertEqual(sc.name, className) 

195 self.assertFalse(sc.components) 

196 self.assertEqual(sc.pytype, PythonType) 

197 self.assertIn(sc, factory) 

198 newclass2 = StorageClass("Temporary2", pytype=str) 

199 self.assertNotIn(newclass2, factory) 

200 factory.registerStorageClass(newclass2) 

201 self.assertIn(newclass2, factory) 

202 self.assertIn("Temporary2", factory) 

203 self.assertNotIn("Temporary3", factory) 

204 self.assertNotIn({}, factory) 

205 

206 # Make sure iterators work. 

207 keys = set(factory.keys()) 

208 self.assertIn("Temporary2", keys) 

209 

210 iterkeys = set(factory) 

211 self.assertEqual(keys, iterkeys) 

212 

213 values = set(factory.values()) 

214 self.assertIn(sc, values) 

215 self.assertEqual(len(factory), len(values)) 

216 

217 external = dict(factory.items()) 

218 self.assertIn("Temporary2", external) 

219 

220 # Make sure we can't register a storage class with the same name 

221 # but different values 

222 newclass3 = StorageClass("Temporary2", pytype=dict) 

223 with self.assertRaises(ValueError) as cm: 

224 factory.registerStorageClass(newclass3) 

225 self.assertIn("pytype='dict'", str(cm.exception)) 

226 with self.assertRaises(ValueError) as cm: 

227 factory.registerStorageClass(newclass3, msg="error string") 

228 self.assertIn("error string", str(cm.exception)) 

229 

230 factory._unregisterStorageClass(newclass3.name) 

231 self.assertNotIn(newclass3, factory) 

232 self.assertNotIn(newclass3.name, factory) 

233 factory.registerStorageClass(newclass3) 

234 self.assertIn(newclass3, factory) 

235 self.assertIn(newclass3.name, factory) 

236 

237 # Check you can silently insert something that is already there 

238 factory.registerStorageClass(newclass3) 

239 

240 def testFactoryFind(self): 

241 # Finding a storage class can involve doing lots of slow imports so 

242 # this is a separate test. 

243 factory = StorageClassFactory() 

244 className = "PythonType3" 

245 newclass = StorageClass(className, pytype=PythonType3) 

246 factory.registerStorageClass(newclass) 

247 sc = factory.getStorageClass(className) 

248 

249 # Can we find a storage class from a type. 

250 new_sc = factory.findStorageClass(PythonType3) 

251 self.assertEqual(new_sc, sc) 

252 

253 # Now with slow mode 

254 new_sc = factory.findStorageClass(PythonType3, compare_types=True) 

255 self.assertEqual(new_sc, sc) 

256 

257 # This class will never match. 

258 with self.assertRaises(KeyError): 

259 factory.findStorageClass(PythonType2, compare_types=True) 

260 

261 # Check builtins. 

262 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict")) 

263 

264 def testFactoryConfig(self): 

265 factory = StorageClassFactory() 

266 factory.addFromConfig(StorageClassConfig()) 

267 image = factory.getStorageClass("Image") 

268 imageF = factory.getStorageClass("ImageF") 

269 self.assertIsInstance(imageF, type(image)) 

270 self.assertNotEqual(imageF, image) 

271 

272 # Check component inheritance 

273 exposure = factory.getStorageClass("Exposure") 

274 exposureF = factory.getStorageClass("ExposureF") 

275 self.assertIsInstance(exposureF, type(exposure)) 

276 self.assertIsInstance(exposure.components["image"], type(image)) 

277 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

278 self.assertIsInstance(exposureF.components["image"], type(image)) 

279 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

280 self.assertIn("wcs", exposure.components) 

281 self.assertIn("wcs", exposureF.components) 

282 

283 # Check parameters 

284 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

285 thing1 = factory.getStorageClass("ThingOne") 

286 thing2 = factory.getStorageClass("ThingTwo") 

287 self.assertIsInstance(thing2, type(thing1)) 

288 param1 = thing1.parameters 

289 param2 = thing2.parameters 

290 self.assertIn("param3", thing2.parameters) 

291 self.assertNotIn("param3", thing1.parameters) 

292 param2.remove("param3") 

293 self.assertEqual(param1, param2) 

294 

295 # Check that we can't have a new StorageClass that does not 

296 # inherit from StorageClass 

297 with self.assertRaises(ValueError): 

298 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

299 

300 sc = factory.makeNewStorageClass("ClassName") 

301 self.assertIsInstance(sc(), StorageClass) 

302 

303 def testPickle(self): 

304 """Test that we can pickle storageclasses.""" 

305 className = "TestImage" 

306 sc = StorageClass(className, pytype=dict) 

307 self.assertIsInstance(sc, StorageClass) 

308 self.assertEqual(sc.name, className) 

309 self.assertFalse(sc.components) 

310 sc2 = pickle.loads(pickle.dumps(sc)) 

311 self.assertEqual(sc2, sc) 

312 

313 @classmethod 

314 def _convert_type(cls, data): 

315 # Test helper function. Fail if the list is empty. 

316 if not len(data): 

317 raise RuntimeError("Deliberate failure.") 

318 return {"key": data} 

319 

320 def testConverters(self): 

321 """Test conversion maps.""" 

322 className = "TestConverters" 

323 converters = { 

324 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

325 # Add some entries that will fail to import. 

326 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

327 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

328 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

329 "list": get_full_type_name(self._convert_type), 

330 } 

331 sc = StorageClass(className, pytype=dict, converters=converters) 

332 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

333 sc2 = StorageClass("Test2", pytype=set) 

334 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

335 

336 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

337 # Initially the converter list is not filtered. 

338 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

339 self.assertNotIn("converters", repr(sc2)) 

340 

341 self.assertTrue(sc.can_convert(sc)) 

342 self.assertFalse(sc.can_convert(sc2)) 

343 self.assertTrue(sc.can_convert(sc3)) 

344 

345 # After we've processed the converters the bad ones will no longer 

346 # be reported. 

347 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

348 self.assertEqual(len(sc.converters), 2) 

349 

350 self.assertIsNone(sc.coerce_type(None)) 

351 

352 converted = sc.coerce_type([1, 2, 3]) 

353 self.assertEqual(converted, {"key": [1, 2, 3]}) 

354 

355 # Convert Metrics using a named method converter. 

356 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

357 converted = sc.coerce_type(metric) 

358 self.assertEqual(converted["data"], [1, 2], converted) 

359 

360 # Check that python types matching is allowed. 

361 sc4 = StorageClass("Test2", pytype=set) 

362 self.assertTrue(sc2.can_convert(sc4)) 

363 converted = sc2.coerce_type({1, 2}) 

364 self.assertEqual(converted, {1, 2}) 

365 

366 # Try to coerce a type that is not supported. 

367 with self.assertRaises(TypeError): 

368 sc.coerce_type({1, 2, 3}) 

369 

370 # Coerce something that will fail to convert. 

371 with self.assertLogs(level=logging.ERROR) as cm: 

372 with self.assertRaises(RuntimeError): 

373 sc.coerce_type([]) 

374 self.assertIn("failed to convert type list", cm.output[0]) 

375 

376 

377if __name__ == "__main__": 

378 unittest.main()