Coverage for tests/test_storageClass.py: 12%

227 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-06 10:53 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import logging 

29import os 

30import pickle 

31import unittest 

32 

33from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

34from lsst.daf.butler.tests import MetricsExample 

35from lsst.utils.introspection import get_full_type_name 

36 

37"""Tests related to the StorageClass infrastructure. 

38""" 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43class PythonType: 

44 """A dummy class to test the registry of Python types.""" 

45 

46 pass 

47 

48 

49class PythonType2: 

50 """A dummy class to test the registry of Python types.""" 

51 

52 pass 

53 

54 

55class PythonType3: 

56 """A dummy class to test the registry of Python types.""" 

57 

58 pass 

59 

60 

61class NotCopyable: 

62 """Class with deep copying disabled.""" 

63 

64 def __deepcopy__(self, memo=None): 

65 raise RuntimeError("Can not be copied.") 

66 

67 

68class StorageClassFactoryTestCase(unittest.TestCase): 

69 """Tests of the storage class infrastructure.""" 

70 

71 def testCreation(self): 

72 """Test that we can dynamically create storage class subclasses. 

73 

74 This is critical for testing the factory functions. 

75 """ 

76 className = "TestImage" 

77 sc = StorageClass(className, pytype=dict) 

78 self.assertIsInstance(sc, StorageClass) 

79 self.assertEqual(sc.name, className) 

80 self.assertEqual(str(sc), className) 

81 self.assertFalse(sc.components) 

82 self.assertTrue(sc.validateInstance({})) 

83 self.assertFalse(sc.validateInstance("")) 

84 

85 r = repr(sc) 

86 self.assertIn("StorageClass", r) 

87 self.assertIn(className, r) 

88 self.assertNotIn("parameters", r) 

89 self.assertIn("pytype='dict'", r) 

90 

91 # Ensure we do not have a delegate 

92 with self.assertRaises(TypeError): 

93 sc.delegate() 

94 

95 # Allow no definition of python type 

96 scn = StorageClass(className) 

97 self.assertIs(scn.pytype, object) 

98 

99 # Include some components 

100 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

101 self.assertIn("comp1", scc.components) 

102 r = repr(scc) 

103 self.assertIn("comp1", r) 

104 self.assertIn("lsst.daf.butler.StorageClassDelegate", r) 

105 

106 # Ensure that we have a delegate 

107 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

108 

109 # Check that delegate copy() works. 

110 list1 = [1, 2, 3] 

111 list2 = scc.delegate().copy(list1) 

112 self.assertEqual(list1, list2) 

113 list2.append(4) 

114 self.assertNotEqual(list1, list2) 

115 

116 with self.assertRaises(NotImplementedError): 

117 scc.delegate().copy(NotCopyable()) 

118 

119 # Check we can create a storageClass using the name of an importable 

120 # type. 

121 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory") 

122 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

123 self.assertIn("butler", repr(sc2)) 

124 

125 def testParameters(self): 

126 """Test that we can set parameters and validate them""" 

127 pt = ("a", "b") 

128 ps = {"a", "b"} 

129 pl = ["a", "b"] 

130 for p in (pt, ps, pl): 

131 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

132 self.assertEqual(sc1.parameters, ps) 

133 sc1.validateParameters(p) 

134 

135 sc1.validateParameters() 

136 sc1.validateParameters({"a": None, "b": None}) 

137 sc1.validateParameters( 

138 [ 

139 "a", 

140 ] 

141 ) 

142 with self.assertRaises(KeyError): 

143 sc1.validateParameters({"a", "c"}) 

144 

145 def testEquality(self): 

146 """Test that StorageClass equality works""" 

147 className = "TestImage" 

148 sc1 = StorageClass(className, pytype=dict) 

149 sc2 = StorageClass(className, pytype=dict) 

150 self.assertEqual(sc1, sc2) 

151 sc3 = StorageClass(className + "2", pytype=str) 

152 self.assertNotEqual(sc1, sc3) 

153 

154 # Same StorageClass name but different python type 

155 sc4 = StorageClass(className, pytype=str) 

156 self.assertNotEqual(sc1, sc4) 

157 

158 # Parameters 

159 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

160 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

161 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

162 self.assertEqual(scp, scp1) 

163 self.assertNotEqual(scp, scp2) 

164 

165 # Now with components 

166 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

167 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

168 self.assertEqual(sc5, sc6) 

169 self.assertNotEqual(sc5, sc3) 

170 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

171 self.assertNotEqual(sc5, sc7) 

172 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

173 self.assertNotEqual(sc5, sc8) 

174 sc9 = StorageClass( 

175 "Composite", 

176 pytype=PythonType, 

177 components={"comp1": sc1, "comp2": sc3}, 

178 delegate="lsst.daf.butler.Butler", 

179 ) 

180 self.assertNotEqual(sc5, sc9) 

181 

182 def testTypeEquality(self): 

183 sc1 = StorageClass("Something", pytype=dict) 

184 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

185 self.assertFalse(sc1.is_type(str), repr(sc1)) 

186 

187 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory") 

188 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

189 

190 def testRegistry(self): 

191 """Check that storage classes can be created on the fly and stored 

192 in a registry. 

193 """ 

194 className = "TestImage" 

195 factory = StorageClassFactory() 

196 newclass = StorageClass(className, pytype=PythonType) 

197 factory.registerStorageClass(newclass) 

198 sc = factory.getStorageClass(className) 

199 self.assertIsInstance(sc, StorageClass) 

200 self.assertEqual(sc.name, className) 

201 self.assertFalse(sc.components) 

202 self.assertEqual(sc.pytype, PythonType) 

203 self.assertIn(sc, factory) 

204 newclass2 = StorageClass("Temporary2", pytype=str) 

205 self.assertNotIn(newclass2, factory) 

206 factory.registerStorageClass(newclass2) 

207 self.assertIn(newclass2, factory) 

208 self.assertIn("Temporary2", factory) 

209 self.assertNotIn("Temporary3", factory) 

210 self.assertNotIn({}, factory) 

211 

212 # Make sure iterators work. 

213 keys = set(factory.keys()) 

214 self.assertIn("Temporary2", keys) 

215 

216 iterkeys = set(factory) 

217 self.assertEqual(keys, iterkeys) 

218 

219 values = set(factory.values()) 

220 self.assertIn(sc, values) 

221 self.assertEqual(len(factory), len(values)) 

222 

223 external = dict(factory.items()) 

224 self.assertIn("Temporary2", external) 

225 

226 # Make sure we can't register a storage class with the same name 

227 # but different values 

228 newclass3 = StorageClass("Temporary2", pytype=dict) 

229 with self.assertRaises(ValueError) as cm: 

230 factory.registerStorageClass(newclass3) 

231 self.assertIn("pytype='dict'", str(cm.exception)) 

232 with self.assertRaises(ValueError) as cm: 

233 factory.registerStorageClass(newclass3, msg="error string") 

234 self.assertIn("error string", str(cm.exception)) 

235 

236 factory._unregisterStorageClass(newclass3.name) 

237 self.assertNotIn(newclass3, factory) 

238 self.assertNotIn(newclass3.name, factory) 

239 factory.registerStorageClass(newclass3) 

240 self.assertIn(newclass3, factory) 

241 self.assertIn(newclass3.name, factory) 

242 

243 # Check you can silently insert something that is already there 

244 factory.registerStorageClass(newclass3) 

245 

246 # Reset the factory and check that default items are present 

247 # but the new ones are not. 

248 factory.reset() 

249 self.assertNotIn(newclass3.name, factory) 

250 self.assertIn("StructuredDataDict", factory) 

251 

252 def testFactoryFind(self): 

253 # Finding a storage class can involve doing lots of slow imports so 

254 # this is a separate test. 

255 factory = StorageClassFactory() 

256 className = "PythonType3" 

257 newclass = StorageClass(className, pytype=PythonType3) 

258 factory.registerStorageClass(newclass) 

259 sc = factory.getStorageClass(className) 

260 

261 # Can we find a storage class from a type. 

262 new_sc = factory.findStorageClass(PythonType3) 

263 self.assertEqual(new_sc, sc) 

264 

265 # Now with slow mode 

266 new_sc = factory.findStorageClass(PythonType3, compare_types=True) 

267 self.assertEqual(new_sc, sc) 

268 

269 # This class will never match. 

270 with self.assertRaises(KeyError): 

271 factory.findStorageClass(PythonType2, compare_types=True) 

272 

273 # Check builtins. 

274 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict")) 

275 

276 def testFactoryConfig(self): 

277 factory = StorageClassFactory() 

278 factory.addFromConfig(StorageClassConfig()) 

279 image = factory.getStorageClass("Image") 

280 imageF = factory.getStorageClass("ImageF") 

281 self.assertIsInstance(imageF, type(image)) 

282 self.assertNotEqual(imageF, image) 

283 

284 # Check component inheritance 

285 exposure = factory.getStorageClass("Exposure") 

286 exposureF = factory.getStorageClass("ExposureF") 

287 self.assertIsInstance(exposureF, type(exposure)) 

288 self.assertIsInstance(exposure.components["image"], type(image)) 

289 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

290 self.assertIsInstance(exposureF.components["image"], type(image)) 

291 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

292 self.assertIn("wcs", exposure.components) 

293 self.assertIn("wcs", exposureF.components) 

294 

295 # Check parameters 

296 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

297 thing1 = factory.getStorageClass("ThingOne") 

298 thing2 = factory.getStorageClass("ThingTwo") 

299 self.assertIsInstance(thing2, type(thing1)) 

300 param1 = thing1.parameters 

301 param2 = thing2.parameters 

302 self.assertIn("param3", thing2.parameters) 

303 self.assertNotIn("param3", thing1.parameters) 

304 param2.remove("param3") 

305 self.assertEqual(param1, param2) 

306 

307 # Check that we can't have a new StorageClass that does not 

308 # inherit from StorageClass 

309 with self.assertRaises(ValueError): 

310 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

311 

312 sc = factory.makeNewStorageClass("ClassName") 

313 self.assertIsInstance(sc(), StorageClass) 

314 

315 def testPickle(self): 

316 """Test that we can pickle storageclasses.""" 

317 className = "TestImage" 

318 sc = StorageClass(className, pytype=dict) 

319 self.assertIsInstance(sc, StorageClass) 

320 self.assertEqual(sc.name, className) 

321 self.assertFalse(sc.components) 

322 sc2 = pickle.loads(pickle.dumps(sc)) 

323 self.assertEqual(sc2, sc) 

324 

325 @classmethod 

326 def _convert_type(cls, data): 

327 # Test helper function. Fail if the list is empty. 

328 if not len(data): 

329 raise RuntimeError("Deliberate failure.") 

330 return {"key": data} 

331 

332 def testConverters(self): 

333 """Test conversion maps.""" 

334 className = "TestConverters" 

335 converters = { 

336 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

337 # Add some entries that will fail to import. 

338 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

339 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

340 "lsst.daf.butler.Butler": "lsst.daf.butler.location.__all__", 

341 "list": get_full_type_name(self._convert_type), 

342 } 

343 sc = StorageClass(className, pytype=dict, converters=converters) 

344 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

345 sc2 = StorageClass("Test2", pytype=set) 

346 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

347 

348 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

349 # Initially the converter list is not filtered. 

350 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

351 self.assertNotIn("converters", repr(sc2)) 

352 

353 self.assertTrue(sc.can_convert(sc)) 

354 self.assertFalse(sc.can_convert(sc2)) 

355 self.assertTrue(sc.can_convert(sc3)) 

356 

357 # After we've processed the converters the bad ones will no longer 

358 # be reported. 

359 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

360 self.assertEqual(len(sc.converters), 2) 

361 

362 self.assertIsNone(sc.coerce_type(None)) 

363 

364 converted = sc.coerce_type([1, 2, 3]) 

365 self.assertEqual(converted, {"key": [1, 2, 3]}) 

366 

367 # Convert Metrics using a named method converter. 

368 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

369 converted = sc.coerce_type(metric) 

370 self.assertEqual(converted["data"], [1, 2], converted) 

371 

372 # Check that python types matching is allowed. 

373 sc4 = StorageClass("Test2", pytype=set) 

374 self.assertTrue(sc2.can_convert(sc4)) 

375 converted = sc2.coerce_type({1, 2}) 

376 self.assertEqual(converted, {1, 2}) 

377 

378 # Try to coerce a type that is not supported. 

379 with self.assertRaises(TypeError): 

380 sc.coerce_type({1, 2, 3}) 

381 

382 # Coerce something that will fail to convert. 

383 with self.assertLogs(level=logging.ERROR) as cm: 

384 with self.assertRaises(RuntimeError): 

385 sc.coerce_type([]) 

386 self.assertIn("failed to convert type list", cm.output[0]) 

387 

388 

389if __name__ == "__main__": 

390 unittest.main()