Coverage for tests/test_storageClass.py: 13%

218 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-26 02:48 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28import logging 

29import os 

30import pickle 

31import unittest 

32 

33from lsst.daf.butler import StorageClass, StorageClassConfig, StorageClassDelegate, StorageClassFactory 

34from lsst.daf.butler.tests import MetricsExample 

35from lsst.utils.introspection import get_full_type_name 

36 

37"""Tests related to the StorageClass infrastructure. 

38""" 

39 

40TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

41 

42 

43class PythonType: 

44 """A dummy class to test the registry of Python types.""" 

45 

46 pass 

47 

48 

49class PythonType2: 

50 """A dummy class to test the registry of Python types.""" 

51 

52 pass 

53 

54 

55class PythonType3: 

56 """A dummy class to test the registry of Python types.""" 

57 

58 pass 

59 

60 

61class NotCopyable: 

62 """Class with deep copying disabled.""" 

63 

64 def __deepcopy__(self, memo=None): 

65 raise RuntimeError("Can not be copied.") 

66 

67 

68class StorageClassFactoryTestCase(unittest.TestCase): 

69 """Tests of the storage class infrastructure.""" 

70 

71 def testCreation(self): 

72 """Test that we can dynamically create storage class subclasses. 

73 

74 This is critical for testing the factory functions. 

75 """ 

76 className = "TestImage" 

77 sc = StorageClass(className, pytype=dict) 

78 self.assertIsInstance(sc, StorageClass) 

79 self.assertEqual(sc.name, className) 

80 self.assertEqual(str(sc), className) 

81 self.assertFalse(sc.components) 

82 self.assertTrue(sc.validateInstance({})) 

83 self.assertFalse(sc.validateInstance("")) 

84 

85 r = repr(sc) 

86 self.assertIn("StorageClass", r) 

87 self.assertIn(className, r) 

88 self.assertNotIn("parameters", r) 

89 self.assertIn("pytype='dict'", r) 

90 

91 # Ensure we do not have a delegate 

92 with self.assertRaises(TypeError): 

93 sc.delegate() 

94 

95 # Allow no definition of python type 

96 scn = StorageClass(className) 

97 self.assertIs(scn.pytype, object) 

98 

99 # Include some components 

100 scc = StorageClass(className, pytype=PythonType, components={"comp1": sc, "comp2": sc}) 

101 self.assertIn("comp1", scc.components) 

102 r = repr(scc) 

103 self.assertIn("comp1", r) 

104 self.assertIn("lsst.daf.butler.StorageClassDelegate", r) 

105 

106 # Ensure that we have a delegate 

107 self.assertIsInstance(scc.delegate(), StorageClassDelegate) 

108 

109 # Check that delegate copy() works. 

110 list1 = [1, 2, 3] 

111 list2 = scc.delegate().copy(list1) 

112 self.assertEqual(list1, list2) 

113 list2.append(4) 

114 self.assertNotEqual(list1, list2) 

115 

116 with self.assertRaises(NotImplementedError): 

117 scc.delegate().copy(NotCopyable()) 

118 

119 # Check we can create a storageClass using the name of an importable 

120 # type. 

121 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory") 

122 self.assertIsInstance(sc2.pytype(), StorageClassFactory) 

123 self.assertIn("butler", repr(sc2)) 

124 

125 def testParameters(self): 

126 """Test that we can set parameters and validate them""" 

127 pt = ("a", "b") 

128 ps = {"a", "b"} 

129 pl = ["a", "b"] 

130 for p in (pt, ps, pl): 

131 sc1 = StorageClass("ParamClass", pytype=dict, parameters=p) 

132 self.assertEqual(sc1.parameters, ps) 

133 sc1.validateParameters(p) 

134 

135 sc1.validateParameters() 

136 sc1.validateParameters({"a": None, "b": None}) 

137 sc1.validateParameters( 

138 [ 

139 "a", 

140 ] 

141 ) 

142 with self.assertRaises(KeyError): 

143 sc1.validateParameters({"a", "c"}) 

144 

145 def testEquality(self): 

146 """Test that StorageClass equality works""" 

147 className = "TestImage" 

148 sc1 = StorageClass(className, pytype=dict) 

149 sc2 = StorageClass(className, pytype=dict) 

150 self.assertEqual(sc1, sc2) 

151 sc3 = StorageClass(className + "2", pytype=str) 

152 self.assertNotEqual(sc1, sc3) 

153 

154 # Same StorageClass name but different python type 

155 sc4 = StorageClass(className, pytype=str) 

156 self.assertNotEqual(sc1, sc4) 

157 

158 # Parameters 

159 scp = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

160 scp1 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "c"]) 

161 scp2 = StorageClass("Params", pytype=PythonType, parameters=["a", "b", "d", "e"]) 

162 self.assertEqual(scp, scp1) 

163 self.assertNotEqual(scp, scp2) 

164 

165 # Now with components 

166 sc5 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

167 sc6 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc1, "comp2": sc3}) 

168 self.assertEqual(sc5, sc6) 

169 self.assertNotEqual(sc5, sc3) 

170 sc7 = StorageClass("Composite", pytype=PythonType, components={"comp1": sc4, "comp2": sc3}) 

171 self.assertNotEqual(sc5, sc7) 

172 sc8 = StorageClass("Composite", pytype=PythonType, components={"comp2": sc3, "comp3": sc3}) 

173 self.assertNotEqual(sc5, sc8) 

174 sc9 = StorageClass( 

175 "Composite", 

176 pytype=PythonType, 

177 components={"comp1": sc1, "comp2": sc3}, 

178 delegate="lsst.daf.butler.Butler", 

179 ) 

180 self.assertNotEqual(sc5, sc9) 

181 

182 def testTypeEquality(self): 

183 sc1 = StorageClass("Something", pytype=dict) 

184 self.assertTrue(sc1.is_type(dict), repr(sc1)) 

185 self.assertFalse(sc1.is_type(str), repr(sc1)) 

186 

187 sc2 = StorageClass("TestImage2", "lsst.daf.butler.StorageClassFactory") 

188 self.assertTrue(sc2.is_type(StorageClassFactory), repr(sc2)) 

189 

190 def testRegistry(self): 

191 """Check that storage classes can be created on the fly and stored 

192 in a registry. 

193 """ 

194 className = "TestImage" 

195 factory = StorageClassFactory() 

196 newclass = StorageClass(className, pytype=PythonType) 

197 factory.registerStorageClass(newclass) 

198 sc = factory.getStorageClass(className) 

199 self.assertIsInstance(sc, StorageClass) 

200 self.assertEqual(sc.name, className) 

201 self.assertFalse(sc.components) 

202 self.assertEqual(sc.pytype, PythonType) 

203 self.assertIn(sc, factory) 

204 newclass2 = StorageClass("Temporary2", pytype=str) 

205 self.assertNotIn(newclass2, factory) 

206 factory.registerStorageClass(newclass2) 

207 self.assertIn(newclass2, factory) 

208 self.assertIn("Temporary2", factory) 

209 self.assertNotIn("Temporary3", factory) 

210 self.assertNotIn({}, factory) 

211 

212 # Make sure we can't register a storage class with the same name 

213 # but different values 

214 newclass3 = StorageClass("Temporary2", pytype=dict) 

215 with self.assertRaises(ValueError) as cm: 

216 factory.registerStorageClass(newclass3) 

217 self.assertIn("pytype='dict'", str(cm.exception)) 

218 with self.assertRaises(ValueError) as cm: 

219 factory.registerStorageClass(newclass3, msg="error string") 

220 self.assertIn("error string", str(cm.exception)) 

221 

222 factory._unregisterStorageClass(newclass3.name) 

223 self.assertNotIn(newclass3, factory) 

224 self.assertNotIn(newclass3.name, factory) 

225 factory.registerStorageClass(newclass3) 

226 self.assertIn(newclass3, factory) 

227 self.assertIn(newclass3.name, factory) 

228 

229 # Check you can silently insert something that is already there 

230 factory.registerStorageClass(newclass3) 

231 

232 # Reset the factory and check that default items are present 

233 # but the new ones are not. 

234 factory.reset() 

235 self.assertNotIn(newclass3.name, factory) 

236 self.assertIn("StructuredDataDict", factory) 

237 

238 def testFactoryFind(self): 

239 # Finding a storage class can involve doing lots of slow imports so 

240 # this is a separate test. 

241 factory = StorageClassFactory() 

242 className = "PythonType3" 

243 newclass = StorageClass(className, pytype=PythonType3) 

244 factory.registerStorageClass(newclass) 

245 sc = factory.getStorageClass(className) 

246 

247 # Can we find a storage class from a type. 

248 new_sc = factory.findStorageClass(PythonType3) 

249 self.assertEqual(new_sc, sc) 

250 

251 # Now with slow mode 

252 new_sc = factory.findStorageClass(PythonType3, compare_types=True) 

253 self.assertEqual(new_sc, sc) 

254 

255 # This class will never match. 

256 with self.assertRaises(KeyError): 

257 factory.findStorageClass(PythonType2, compare_types=True) 

258 

259 # Check builtins. 

260 self.assertEqual(factory.findStorageClass(dict), factory.getStorageClass("StructuredDataDict")) 

261 

262 def testFactoryConfig(self): 

263 factory = StorageClassFactory() 

264 factory.addFromConfig(StorageClassConfig()) 

265 image = factory.getStorageClass("Image") 

266 imageF = factory.getStorageClass("ImageF") 

267 self.assertIsInstance(imageF, type(image)) 

268 self.assertNotEqual(imageF, image) 

269 

270 # Check component inheritance 

271 exposure = factory.getStorageClass("Exposure") 

272 exposureF = factory.getStorageClass("ExposureF") 

273 self.assertIsInstance(exposureF, type(exposure)) 

274 self.assertIsInstance(exposure.components["image"], type(image)) 

275 self.assertNotIsInstance(exposure.components["image"], type(imageF)) 

276 self.assertIsInstance(exposureF.components["image"], type(image)) 

277 self.assertIsInstance(exposureF.components["image"], type(imageF)) 

278 self.assertIn("wcs", exposure.components) 

279 self.assertIn("wcs", exposureF.components) 

280 

281 # Check parameters 

282 factory.addFromConfig(os.path.join(TESTDIR, "config", "basic", "storageClasses.yaml")) 

283 thing1 = factory.getStorageClass("ThingOne") 

284 thing2 = factory.getStorageClass("ThingTwo") 

285 self.assertIsInstance(thing2, type(thing1)) 

286 param1 = thing1.parameters 

287 param2 = thing2.parameters 

288 self.assertIn("param3", thing2.parameters) 

289 self.assertNotIn("param3", thing1.parameters) 

290 param2.remove("param3") 

291 self.assertEqual(param1, param2) 

292 

293 # Check that we can't have a new StorageClass that does not 

294 # inherit from StorageClass 

295 with self.assertRaises(ValueError): 

296 factory.makeNewStorageClass("ClassName", baseClass=StorageClassFactory) 

297 

298 sc = factory.makeNewStorageClass("ClassName") 

299 self.assertIsInstance(sc(), StorageClass) 

300 

301 def testPickle(self): 

302 """Test that we can pickle storageclasses.""" 

303 className = "TestImage" 

304 sc = StorageClass(className, pytype=dict) 

305 self.assertIsInstance(sc, StorageClass) 

306 self.assertEqual(sc.name, className) 

307 self.assertFalse(sc.components) 

308 sc2 = pickle.loads(pickle.dumps(sc)) 

309 self.assertEqual(sc2, sc) 

310 

311 @classmethod 

312 def _convert_type(cls, data): 

313 # Test helper function. Fail if the list is empty. 

314 if not len(data): 

315 raise RuntimeError("Deliberate failure.") 

316 return {"key": data} 

317 

318 def testConverters(self): 

319 """Test conversion maps.""" 

320 className = "TestConverters" 

321 converters = { 

322 "lsst.daf.butler.tests.MetricsExample": "lsst.daf.butler.tests.MetricsExample.exportAsDict", 

323 # Add some entries that will fail to import. 

324 "lsst.daf.butler.bad.type": "lsst.daf.butler.tests.MetricsExampleModel.from_metrics", 

325 "lsst.daf.butler.tests.MetricsExampleModel": "lsst.daf.butler.bad.function", 

326 "lsst.daf.butler.Butler": "lsst.daf.butler.location.__all__", 

327 "list": get_full_type_name(self._convert_type), 

328 } 

329 sc = StorageClass(className, pytype=dict, converters=converters) 

330 self.assertEqual(len(sc.converters), 5) # Pre-filtering 

331 sc2 = StorageClass("Test2", pytype=set) 

332 sc3 = StorageClass("Test3", pytype="lsst.daf.butler.tests.MetricsExample") 

333 

334 self.assertIn("lsst.daf.butler.tests.MetricsExample", repr(sc)) 

335 # Initially the converter list is not filtered. 

336 self.assertIn("lsst.daf.butler.bad.type", repr(sc)) 

337 self.assertNotIn("converters", repr(sc2)) 

338 

339 self.assertTrue(sc.can_convert(sc)) 

340 self.assertFalse(sc.can_convert(sc2)) 

341 self.assertTrue(sc.can_convert(sc3)) 

342 

343 # After we've processed the converters the bad ones will no longer 

344 # be reported. 

345 self.assertNotIn("lsst.daf.butler.bad.type", repr(sc)) 

346 self.assertEqual(len(sc.converters), 2) 

347 

348 self.assertIsNone(sc.coerce_type(None)) 

349 

350 converted = sc.coerce_type([1, 2, 3]) 

351 self.assertEqual(converted, {"key": [1, 2, 3]}) 

352 

353 # Convert Metrics using a named method converter. 

354 metric = MetricsExample(summary={"a": 1}, data=[1, 2], output={"c": "e"}) 

355 converted = sc.coerce_type(metric) 

356 self.assertEqual(converted["data"], [1, 2], converted) 

357 

358 # Check that python types matching is allowed. 

359 sc4 = StorageClass("Test2", pytype=set) 

360 self.assertTrue(sc2.can_convert(sc4)) 

361 converted = sc2.coerce_type({1, 2}) 

362 self.assertEqual(converted, {1, 2}) 

363 

364 # Try to coerce a type that is not supported. 

365 with self.assertRaises(TypeError): 

366 sc.coerce_type({1, 2, 3}) 

367 

368 # Coerce something that will fail to convert. 

369 with self.assertLogs(level=logging.ERROR) as cm: 

370 with self.assertRaises(RuntimeError): 

371 sc.coerce_type([]) 

372 self.assertIn("failed to convert type list", cm.output[0]) 

373 

374 

375if __name__ == "__main__": 

376 unittest.main()