Coverage for tests/test_storageClass.py: 12%

224 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 09:13 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import logging 

23import os 

24import pickle 

25import unittest 

26 

27from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

28from lsst.daf.butler.tests import MetricsExample 

29from lsst.utils.introspection import get_full_type_name 

30 

31"""Tests related to the StorageClass infrastructure. 

32""" 

33 

34TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

35 

36 

37class PythonType: 

38 """A dummy class to test the registry of Python types.""" 

39 

40 pass 

41 

42 

43class PythonType2: 

44 """A dummy class to test the registry of Python types.""" 

45 

46 pass 

47 

48 

49class PythonType3: 

50 """A dummy class to test the registry of Python types.""" 

51 

52 pass 

53 

54 

55class NotCopyable: 

56 def __deepcopy__(self, memo=None): 

57 raise RuntimeError("Can not be copied.") 

58 

59 

60class StorageClassFactoryTestCase(unittest.TestCase): 

61 """Tests of the storage class infrastructure.""" 

62 

63 def testCreation(self): 

64 """Test that we can dynamically create storage class subclasses. 

65 

66 This is critical for testing the factory functions.""" 

67 className = "TestImage" 

68 sc = StorageClass(className, pytype=dict) 

69 self.assertIsInstance(sc, StorageClass) 

70 self.assertEqual(sc.name, className) 

71 self.assertEqual(str(sc), className) 

72 self.assertFalse(sc.components) 

73 self.assertTrue(sc.validateInstance({})) 

74 self.assertFalse(sc.validateInstance("")) 

75 

76 r = repr(sc) 

77 self.assertIn("StorageClass", r) 

78 self.assertIn(className, r) 

79 self.assertNotIn("parameters", r) 

80 self.assertIn("pytype='dict'", r) 

81 

82 # Ensure we do not have a delegate 

83 with self.assertRaises(TypeError): 

84 sc.delegate() 

85 

86 # Allow no definition of python type 

87 scn = StorageClass(className) 

88 self.assertIs(scn.pytype, object) 

89 

90 # Include some components 

91 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

92 self.assertIn("comp1", scc.components) 

93 r = repr(scc) 

94 self.assertIn("comp1", r) 

95 self.assertIn("lsst.daf.butler.core.storageClassDelegate.StorageClassDelegate", r) 

96 

97 # Ensure that we have a delegate 

98 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

99 

100 # Check that delegate copy() works. 

101 list1 = [1, 2, 3] 

102 list2 = scc.delegate().copy(list1) 

103 self.assertEqual(list1, list2) 

104 list2.append(4) 

105 self.assertNotEqual(list1, list2) 

106 

107 with self.assertRaises(NotImplementedError): 

108 scc.delegate().copy(NotCopyable()) 

109 

110 # Check we can create a storageClass using the name of an importable 

111 # type. 

112 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

113 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

114 self.assertIn("butler.core", repr(sc2)) 

115 

116 def testParameters(self): 

117 """Test that we can set parameters and validate them""" 

118 pt = ("a", "b") 

119 ps = {"a", "b"} 

120 pl = ["a", "b"] 

121 for p in (pt, ps, pl): 

122 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

123 self.assertEqual(sc1.parameters, ps) 

124 sc1.validateParameters(p) 

125 

126 sc1.validateParameters() 

127 sc1.validateParameters({"a": None, "b": None}) 

128 sc1.validateParameters( 

129 [ 

130 "a", 

131 ] 

132 ) 

133 with self.assertRaises(KeyError): 

134 sc1.validateParameters({"a", "c"}) 

135 

136 def testEquality(self): 

137 """Test that StorageClass equality works""" 

138 className = "TestImage" 

139 sc1 = StorageClass(className, pytype=dict) 

140 sc2 = StorageClass(className, pytype=dict) 

141 self.assertEqual(sc1, sc2) 

142 sc3 = StorageClass(className + "2", pytype=str) 

143 self.assertNotEqual(sc1, sc3) 

144 

145 # Same StorageClass name but different python type 

146 sc4 = StorageClass(className, pytype=str) 

147 self.assertNotEqual(sc1, sc4) 

148 

149 # Parameters 

150 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

151 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

152 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

153 self.assertEqual(scp, scp1) 

154 self.assertNotEqual(scp, scp2) 

155 

156 # Now with components 

157 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

158 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

159 self.assertEqual(sc5, sc6) 

160 self.assertNotEqual(sc5, sc3) 

161 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

162 self.assertNotEqual(sc5, sc7) 

163 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

164 self.assertNotEqual(sc5, sc8) 

165 sc9 = StorageClass( 

166 "Composite", 

167 pytype=PythonType, 

168 components={"comp1": sc1, "comp2": sc3}, 

169 delegate="lsst.daf.butler.Butler", 

170 ) 

171 self.assertNotEqual(sc5, sc9) 

172 

173 def testTypeEquality(self): 

174 sc1 = StorageClass("Something", pytype=dict) 

175 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

176 self.assertFalse(sc1.is_type(str), repr(sc1)) 

177 

178 sc2 = StorageClass("TestImage2", "lsst.daf.butler.core.storageClass.StorageClassFactory") 

179 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

180 

181 def testRegistry(self): 

182 """Check that storage classes can be created on the fly and stored 

183 in a registry.""" 

184 className = "TestImage" 

185 factory = StorageClassFactory() 

186 newclass = StorageClass(className, pytype=PythonType) 

187 factory.registerStorageClass(newclass) 

188 sc = factory.getStorageClass(className) 

189 self.assertIsInstance(sc, StorageClass) 

190 self.assertEqual(sc.name, className) 

191 self.assertFalse(sc.components) 

192 self.assertEqual(sc.pytype, PythonType) 

193 self.assertIn(sc, factory) 

194 newclass2 = StorageClass("Temporary2", pytype=str) 

195 self.assertNotIn(newclass2, factory) 

196 factory.registerStorageClass(newclass2) 

197 self.assertIn(newclass2, factory) 

198 self.assertIn("Temporary2", factory) 

199 self.assertNotIn("Temporary3", factory) 

200 self.assertNotIn({}, factory) 

201 

202 # Make sure iterators work. 

203 keys = set(factory.keys()) 

204 self.assertIn("Temporary2", keys) 

205 

206 iterkeys = {k for k in factory} 

207 self.assertEqual(keys, iterkeys) 

208 

209 values = set(factory.values()) 

210 self.assertIn(sc, values) 

211 self.assertEqual(len(factory), len(values)) 

212 

213 external = {k: v for k, v in factory.items()} 

214 self.assertIn("Temporary2", external) 

215 

216 # Make sure we can't register a storage class with the same name 

217 # but different values 

218 newclass3 = StorageClass("Temporary2", pytype=dict) 

219 with self.assertRaises(ValueError) as cm: 

220 factory.registerStorageClass(newclass3) 

221 self.assertIn("pytype='dict'", str(cm.exception)) 

222 with self.assertRaises(ValueError) as cm: 

223 factory.registerStorageClass(newclass3, msg="error string") 

224 self.assertIn("error string", str(cm.exception)) 

225 

226 factory._unregisterStorageClass(newclass3.name) 

227 self.assertNotIn(newclass3, factory) 

228 self.assertNotIn(newclass3.name, factory) 

229 factory.registerStorageClass(newclass3) 

230 self.assertIn(newclass3, factory) 

231 self.assertIn(newclass3.name, factory) 

232 

233 # Check you can silently insert something that is already there 

234 factory.registerStorageClass(newclass3) 

235 

236 def testFactoryFind(self): 

237 # Finding a storage class can involve doing lots of slow imports so 

238 # this is a separate test. 

239 factory = StorageClassFactory() 

240 className = "PythonType3" 

241 newclass = StorageClass(className, pytype=PythonType3) 

242 factory.registerStorageClass(newclass) 

243 sc = factory.getStorageClass(className) 

244 

245 # Can we find a storage class from a type. 

246 new_sc = factory.findStorageClass(PythonType3) 

247 self.assertEqual(new_sc, sc) 

248 

249 # Now with slow mode 

250 new_sc = factory.findStorageClass(PythonType3, compare_types=True) 

251 self.assertEqual(new_sc, sc) 

252 

253 # This class will never match. 

254 with self.assertRaises(KeyError): 

255 factory.findStorageClass(PythonType2, compare_types=True) 

256 

257 # Check builtins. 

258 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict")) 

259 

260 def testFactoryConfig(self): 

261 factory = StorageClassFactory() 

262 factory.addFromConfig(StorageClassConfig()) 

263 image = factory.getStorageClass("Image") 

264 imageF = factory.getStorageClass("ImageF") 

265 self.assertIsInstance(imageF, type(image)) 

266 self.assertNotEqual(imageF, image) 

267 

268 # Check component inheritance 

269 exposure = factory.getStorageClass("Exposure") 

270 exposureF = factory.getStorageClass("ExposureF") 

271 self.assertIsInstance(exposureF, type(exposure)) 

272 self.assertIsInstance(exposure.components["image"], type(image)) 

273 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

274 self.assertIsInstance(exposureF.components["image"], type(image)) 

275 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

276 self.assertIn("wcs", exposure.components) 

277 self.assertIn("wcs", exposureF.components) 

278 

279 # Check parameters 

280 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

281 thing1 = factory.getStorageClass("ThingOne") 

282 thing2 = factory.getStorageClass("ThingTwo") 

283 self.assertIsInstance(thing2, type(thing1)) 

284 param1 = thing1.parameters 

285 param2 = thing2.parameters 

286 self.assertIn("param3", thing2.parameters) 

287 self.assertNotIn("param3", thing1.parameters) 

288 param2.remove("param3") 

289 self.assertEqual(param1, param2) 

290 

291 # Check that we can't have a new StorageClass that does not 

292 # inherit from StorageClass 

293 with self.assertRaises(ValueError): 

294 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

295 

296 sc = factory.makeNewStorageClass("ClassName") 

297 self.assertIsInstance(sc(), StorageClass) 

298 

299 def testPickle(self): 

300 """Test that we can pickle storageclasses.""" 

301 className = "TestImage" 

302 sc = StorageClass(className, pytype=dict) 

303 self.assertIsInstance(sc, StorageClass) 

304 self.assertEqual(sc.name, className) 

305 self.assertFalse(sc.components) 

306 sc2 = pickle.loads(pickle.dumps(sc)) 

307 self.assertEqual(sc2, sc) 

308 

309 @classmethod 

310 def _convert_type(cls, data): 

311 # Test helper function. Fail if the list is empty. 

312 if not len(data): 

313 raise RuntimeError("Deliberate failure.") 

314 return {"key": data} 

315 

316 def testConverters(self): 

317 """Test conversion maps.""" 

318 

319 className = "TestConverters" 

320 converters = { 

321 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

322 # Add some entries that will fail to import. 

323 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

324 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

325 "lsst.daf.butler.Butler": "lsst.daf.butler.core.location.__all__", 

326 "list": get_full_type_name(self._convert_type), 

327 } 

328 sc = StorageClass(className, pytype=dict, converters=converters) 

329 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

330 sc2 = StorageClass("Test2", pytype=set) 

331 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

332 

333 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

334 # Initially the converter list is not filtered. 

335 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

336 self.assertNotIn("converters", repr(sc2)) 

337 

338 self.assertTrue(sc.can_convert(sc)) 

339 self.assertFalse(sc.can_convert(sc2)) 

340 self.assertTrue(sc.can_convert(sc3)) 

341 

342 # After we've processed the converters the bad ones will no longer 

343 # be reported. 

344 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

345 self.assertEqual(len(sc.converters), 2) 

346 

347 self.assertIsNone(sc.coerce_type(None)) 

348 

349 converted = sc.coerce_type([1, 2, 3]) 

350 self.assertEqual(converted, {"key": [1, 2, 3]}) 

351 

352 # Convert Metrics using a named method converter. 

353 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

354 converted = sc.coerce_type(metric) 

355 self.assertEqual(converted["data"], [1, 2], converted) 

356 

357 # Check that python types matching is allowed. 

358 sc4 = StorageClass("Test2", pytype=set) 

359 self.assertTrue(sc2.can_convert(sc4)) 

360 converted = sc2.coerce_type({1, 2}) 

361 self.assertEqual(converted, {1, 2}) 

362 

363 # Try to coerce a type that is not supported. 

364 with self.assertRaises(TypeError): 

365 sc.coerce_type({1, 2, 3}) 

366 

367 # Coerce something that will fail to convert. 

368 with self.assertLogs(level=logging.ERROR) as cm: 

369 with self.assertRaises(RuntimeError): 

370 sc.coerce_type([]) 

371 self.assertIn("failed to convert type list", cm.output[0]) 

372 

373 

374if __name__ == "__main__": 

375 unittest.main()