Coverage for tests/test_repositoryCfg.py: 29%

138 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-01-05 02:50 -0800

1# 

2# LSST Data Management System 

3# Copyright 2016 LSST Corporation. 

4# 

5# This product includes software developed by the 

6# LSST Project (http://www.lsst.org/). 

7# 

8# This program is free software: you can redistribute it and/or modify 

9# it under the terms of the GNU General Public License as published by 

10# the Free Software Foundation, either version 3 of the License, or 

11# (at your option) any later version. 

12# 

13# This program is distributed in the hope that it will be useful, 

14# but WITHOUT ANY WARRANTY; without even the implied warranty of 

15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

16# GNU General Public License for more details. 

17# 

18# You should have received a copy of the LSST License Statement and 

19# the GNU General Public License along with this program. If not, 

20# see <http://www.lsstcorp.org/LegalNotices/>. 

21# 

22import os 

23import shutil 

24import tempfile 

25import unittest 

26import yaml 

27 

28import lsst.daf.persistence as dp 

29import lsst.daf.persistence.test as dpTest 

30import lsst.utils.tests 

31 

32# Define the root of the tests relative to this file 

33ROOT = os.path.abspath(os.path.dirname(__file__)) 

34 

35 

36class TestCfgRelationship(unittest.TestCase): 

37 

38 def setUp(self): 

39 self.testDir = tempfile.mkdtemp(dir=ROOT, 

40 prefix="TestCfgRelationship-") 

41 

42 def tearDown(self): 

43 if os.path.exists(self.testDir): 

44 shutil.rmtree(self.testDir) 

45 

46 def testRWModes(self): 

47 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper, 

48 root=self.testDir) 

49 butler = dp.Butler(outputs=args) 

50 # inputs must be read-only or read-write and not write-only 

51 args = dp.RepositoryArgs(mode='r', mapper=dpTest.EmptyTestMapper, 

52 root=self.testDir) 

53 butler = dp.Butler(inputs=args) 

54 args = dp.RepositoryArgs(mode='rw', mapper=dpTest.EmptyTestMapper, 

55 root=self.testDir) 

56 butler = dp.Butler(inputs=args) 

57 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper, 

58 root=self.testDir) 

59 self.assertRaises(RuntimeError, dp.Butler, inputs=args) 

60 

61 # outputs must be write-only or read-write and not read-only 

62 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper, 

63 root=self.testDir) 

64 butler = dp.Butler(outputs=args) 

65 args = dp.RepositoryArgs(mode='rw', mapper=dpTest.EmptyTestMapper, 

66 root=self.testDir) 

67 butler = dp.Butler(outputs=args) 

68 self.assertIsInstance(butler, dp.Butler) 

69 args = dp.RepositoryArgs(mode='r', mapper=dpTest.EmptyTestMapper, 

70 root=self.testDir) 

71 self.assertRaises(RuntimeError, dp.Butler, outputs=args) 

72 

73 def testExistingParents(self): 

74 # parents of inputs should be added to the inputs list 

75 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w', 

76 mapper=dpTest.EmptyTestMapper, 

77 root=os.path.join(self.testDir, 'a'))) 

78 del butler 

79 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

80 outputs=os.path.join(self.testDir, 'b')) 

81 del butler 

82 butler = dp.Butler(inputs=os.path.join(self.testDir, 'b')) 

83 self.assertEqual(len(butler._repos.inputs()), 2) 

84 # verify serach order: 

85 self.assertEqual(butler._repos.inputs()[0].cfg.root, 

86 os.path.join(self.testDir, 'b')) 

87 self.assertEqual(butler._repos.inputs()[1].cfg.root, 

88 os.path.join(self.testDir, 'a')) 

89 self.assertEqual(len(butler._repos.outputs()), 0) 

90 

91 butler = dp.Butler(outputs=dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'b'), 

92 mode='rw')) 

93 # verify serach order: 

94 self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) 

95 self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) 

96 self.assertEqual(len(butler._repos.outputs()), 1) 

97 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) 

98 

99 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

100 outputs=dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'b'), 

101 mode='rw')) 

102 self.assertEqual(len(butler._repos.inputs()), 2) 

103 # verify serach order: 

104 self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b')) 

105 self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a')) 

106 self.assertEqual(len(butler._repos.outputs()), 1) 

107 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) 

108 

109 # parents of write-only outputs must be be listed with the inputs 

110 with self.assertRaises(RuntimeError): 

111 butler = dp.Butler(outputs=os.path.join(self.testDir, 'b')) 

112 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'), 

113 outputs=os.path.join(self.testDir, 'b')) 

114 self.assertEqual(len(butler._repos.inputs()), 1) 

115 self.assertEqual(len(butler._repos.outputs()), 1) 

116 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b')) 

117 

118 # add a new parent to an existing output 

119 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w', 

120 mapper=dpTest.EmptyTestMapper, 

121 root=os.path.join(self.testDir, 'c'))) 

122 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'a'), 

123 os.path.join(self.testDir, 'c')), 

124 outputs=os.path.join(self.testDir, 'b')) 

125 

126 # should raise if the input order gets reversed: 

127 with self.assertRaises(RuntimeError): 

128 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'c'), 

129 os.path.join(self.testDir, 'a')), 

130 outputs=os.path.join(self.testDir, 'b')) 

131 

132 def testSymLinkInPath(self): 

133 """Test that when a symlink is in an output repo path that the repoCfg 

134 stored in the output repo has a parent path from the actual output 

135 lcoation to the input repo.""" 

136 # create a repository at 'a' 

137 repoADir = os.path.join(self.testDir, 'a') 

138 repoBDir = os.path.join(self.testDir, 'b') 

139 butler = dp.Butler(outputs=dp.RepositoryArgs( 

140 mode='w', mapper=dpTest.EmptyTestMapper, 

141 root=repoADir)) 

142 # create a symlink from 'b' to a temporary directory 

143 tmpDir = tempfile.mkdtemp() 

144 os.symlink(tmpDir, repoBDir) 

145 # create an output repo at 'b' with the input repo 'a' 

146 butler = dp.Butler(inputs=repoADir, outputs=repoBDir) 

147 self.assertIsInstance(butler, dp.Butler) 

148 # verify a repoCfg in the tmp dir with a proper parent path from tmp 

149 # to 'a' (not from 'b' to 'a') 

150 cfg = dp.PosixStorage.getRepositoryCfg(tmpDir) 

151 # verify that the parent link gotten via the symlink target is a path 

152 # to A 

153 self.assertEqual(repoADir, cfg.parents[0]) 

154 cfg = dp.PosixStorage.getRepositoryCfg(repoBDir) 

155 # also verify that the parent gotten via the symlink is a path to A 

156 self.assertEqual(repoADir, cfg.parents[0]) 

157 

158 def testStorageRepoCfgCache(self): 

159 """Tests that when a cfg is gotten from storage it is cached.""" 

160 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w', 

161 mapper=dpTest.EmptyTestMapper, 

162 root=os.path.join(self.testDir, 'a'))) 

163 del butler 

164 storage = dp.Storage() 

165 self.assertEqual(0, len(storage.repositoryCfgs)) 

166 cfg = storage.getRepositoryCfg(os.path.join(self.testDir, 'a')) 

167 self.assertEqual(cfg, storage.repositoryCfgs[os.path.join(self.testDir, 'a')]) 

168 

169 

170class TestNestedCfg(unittest.TestCase): 

171 

172 def setUp(self): 

173 self.testDir = tempfile.mkdtemp(dir=ROOT, 

174 prefix="TestNestedCfg-") 

175 

176 def tearDown(self): 

177 if os.path.exists(self.testDir): 

178 shutil.rmtree(self.testDir) 

179 

180 def test(self): 

181 parentCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'parent'), 

182 mapper='lsst.daf.persistence.SomeMapper', 

183 mapperArgs={}, 

184 parents=None, 

185 policy=None) 

186 cfg = dp.RepositoryCfg(root=self.testDir, 

187 mapper='lsst.daf.persistence.SomeMapper', 

188 mapperArgs={}, 

189 parents=[parentCfg], 

190 policy=None) 

191 dp.PosixStorage.putRepositoryCfg(cfg) 

192 reloadedCfg = dp.PosixStorage.getRepositoryCfg(self.testDir) 

193 self.assertEqual(cfg, reloadedCfg) 

194 self.assertEqual(cfg.parents[0], parentCfg) 

195 

196 

197# "fake" repository version 0 

198class RepositoryCfg(yaml.YAMLObject): 

199 yaml_tag = u"!RepositoryCfg_v0" 

200 

201 def __init__(self, mapper, mapperArgs): 

202 self._mapper = mapper 

203 self._mapperArgs = mapperArgs 

204 

205 @staticmethod 

206 def v0Constructor(loader, node): 

207 d = loader.construct_mapping(node) 

208 return dp.RepositoryCfg(root=d['_root'], mapper=None, mapperArgs=None, parents=None, policy=None) 

209 

210 

211loaderList = [yaml.Loader, ] 

212try: 

213 loaderList.append(yaml.UnsafeLoader) 

214except AttributeError: 

215 pass 

216 

217for loader in loaderList: 

218 yaml.add_constructor(u"!RepositoryCfg_v0", RepositoryCfg.v0Constructor, Loader=loader) 

219 

220 

221class TestCfgFileVersion(unittest.TestCase): 

222 """Proof-of-concept test case for a fictitious version 0 of a persisted repository cfg. 

223 """ 

224 

225 def setUp(self): 

226 self.testDir = tempfile.mkdtemp(dir=ROOT, 

227 prefix="TestCfgFileVersion-") 

228 

229 def tearDown(self): 

230 if os.path.exists(self.testDir): 

231 shutil.rmtree(self.testDir) 

232 

233 def test(self): 

234 with open(os.path.join(self.testDir, 'repositoryCfg.yaml'), 'w') as f: 

235 f.write("""!RepositoryCfg_v0 

236 _root: 'foo/bar'""") 

237 cfg = dp.PosixStorage.getRepositoryCfg(self.testDir) 

238 self.assertIsInstance(cfg, dp.RepositoryCfg) 

239 

240 

241class TestExtendParents(unittest.TestCase): 

242 """Test for the RepositoryCfg.extendParents function. 

243 """ 

244 

245 def test(self): 

246 cfg = dp.RepositoryCfg(root='.', mapper='my.mapper.class', mapperArgs=None, 

247 parents=['bar', 'baz'], policy=None) 

248 cfg.extendParents(['bar', 'baz', 'qux']) 

249 with self.assertRaises(dp.ParentsMismatch): 

250 cfg.extendParents(['bar', 'baz', 'corge']) 

251 

252 # todo include a repositoryCfg in parents lists in this test 

253 

254 

255class TestMapperArgsNone(unittest.TestCase): 

256 """Tests that the RepositoryCfg.mapperArgs is converted to a dict if None is passed in. 

257 """ 

258 def test(self): 

259 cfg = dp.RepositoryCfg(root='foo', mapper='foo', mapperArgs=None, parents=None, policy=None) 

260 self.assertIsInstance(cfg.mapperArgs, dict) 

261 

262 

263class MemoryTester(lsst.utils.tests.MemoryTestCase): 

264 pass 

265 

266 

267def setup_module(module): 

268 lsst.utils.tests.init() 

269 

270 

271if __name__ == '__main__': 271 ↛ 272line 271 didn't jump to line 272, because the condition on line 271 was never true

272 lsst.utils.tests.init() 

273 unittest.main()