Coverage for tests/test_repositoryCfg.py: 35%
138 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:20 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:20 -0700
1#
2# LSST Data Management System
3# Copyright 2016 LSST Corporation.
4#
5# This product includes software developed by the
6# LSST Project (http://www.lsst.org/).
7#
8# This program is free software: you can redistribute it and/or modify
9# it under the terms of the GNU General Public License as published by
10# the Free Software Foundation, either version 3 of the License, or
11# (at your option) any later version.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the LSST License Statement and
19# the GNU General Public License along with this program. If not,
20# see <http://www.lsstcorp.org/LegalNotices/>.
21#
22import os
23import shutil
24import tempfile
25import unittest
26import yaml
28import lsst.daf.persistence as dp
29import lsst.daf.persistence.test as dpTest
30import lsst.utils.tests
32# Define the root of the tests relative to this file
33ROOT = os.path.abspath(os.path.dirname(__file__))
36class TestCfgRelationship(unittest.TestCase):
38 def setUp(self):
39 self.testDir = tempfile.mkdtemp(dir=ROOT,
40 prefix="TestCfgRelationship-")
42 def tearDown(self):
43 if os.path.exists(self.testDir):
44 shutil.rmtree(self.testDir)
46 def testRWModes(self):
47 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper,
48 root=self.testDir)
49 butler = dp.Butler(outputs=args)
50 # inputs must be read-only or read-write and not write-only
51 args = dp.RepositoryArgs(mode='r', mapper=dpTest.EmptyTestMapper,
52 root=self.testDir)
53 butler = dp.Butler(inputs=args)
54 args = dp.RepositoryArgs(mode='rw', mapper=dpTest.EmptyTestMapper,
55 root=self.testDir)
56 butler = dp.Butler(inputs=args)
57 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper,
58 root=self.testDir)
59 self.assertRaises(RuntimeError, dp.Butler, inputs=args)
61 # outputs must be write-only or read-write and not read-only
62 args = dp.RepositoryArgs(mode='w', mapper=dpTest.EmptyTestMapper,
63 root=self.testDir)
64 butler = dp.Butler(outputs=args)
65 args = dp.RepositoryArgs(mode='rw', mapper=dpTest.EmptyTestMapper,
66 root=self.testDir)
67 butler = dp.Butler(outputs=args)
68 self.assertIsInstance(butler, dp.Butler)
69 args = dp.RepositoryArgs(mode='r', mapper=dpTest.EmptyTestMapper,
70 root=self.testDir)
71 self.assertRaises(RuntimeError, dp.Butler, outputs=args)
73 def testExistingParents(self):
74 # parents of inputs should be added to the inputs list
75 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w',
76 mapper=dpTest.EmptyTestMapper,
77 root=os.path.join(self.testDir, 'a')))
78 del butler
79 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
80 outputs=os.path.join(self.testDir, 'b'))
81 del butler
82 butler = dp.Butler(inputs=os.path.join(self.testDir, 'b'))
83 self.assertEqual(len(butler._repos.inputs()), 2)
84 # verify serach order:
85 self.assertEqual(butler._repos.inputs()[0].cfg.root,
86 os.path.join(self.testDir, 'b'))
87 self.assertEqual(butler._repos.inputs()[1].cfg.root,
88 os.path.join(self.testDir, 'a'))
89 self.assertEqual(len(butler._repos.outputs()), 0)
91 butler = dp.Butler(outputs=dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'b'),
92 mode='rw'))
93 # verify serach order:
94 self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b'))
95 self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a'))
96 self.assertEqual(len(butler._repos.outputs()), 1)
97 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b'))
99 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
100 outputs=dp.RepositoryArgs(cfgRoot=os.path.join(self.testDir, 'b'),
101 mode='rw'))
102 self.assertEqual(len(butler._repos.inputs()), 2)
103 # verify serach order:
104 self.assertEqual(butler._repos.inputs()[0].cfg.root, os.path.join(self.testDir, 'b'))
105 self.assertEqual(butler._repos.inputs()[1].cfg.root, os.path.join(self.testDir, 'a'))
106 self.assertEqual(len(butler._repos.outputs()), 1)
107 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b'))
109 # parents of write-only outputs must be be listed with the inputs
110 with self.assertRaises(RuntimeError):
111 butler = dp.Butler(outputs=os.path.join(self.testDir, 'b'))
112 butler = dp.Butler(inputs=os.path.join(self.testDir, 'a'),
113 outputs=os.path.join(self.testDir, 'b'))
114 self.assertEqual(len(butler._repos.inputs()), 1)
115 self.assertEqual(len(butler._repos.outputs()), 1)
116 self.assertEqual(butler._repos.outputs()[0].cfg.root, os.path.join(self.testDir, 'b'))
118 # add a new parent to an existing output
119 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w',
120 mapper=dpTest.EmptyTestMapper,
121 root=os.path.join(self.testDir, 'c')))
122 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'a'),
123 os.path.join(self.testDir, 'c')),
124 outputs=os.path.join(self.testDir, 'b'))
126 # should raise if the input order gets reversed:
127 with self.assertRaises(RuntimeError):
128 butler = dp.Butler(inputs=(os.path.join(self.testDir, 'c'),
129 os.path.join(self.testDir, 'a')),
130 outputs=os.path.join(self.testDir, 'b'))
132 def testSymLinkInPath(self):
133 """Test that when a symlink is in an output repo path that the repoCfg
134 stored in the output repo has a parent path from the actual output
135 lcoation to the input repo."""
136 # create a repository at 'a'
137 repoADir = os.path.join(self.testDir, 'a')
138 repoBDir = os.path.join(self.testDir, 'b')
139 butler = dp.Butler(outputs=dp.RepositoryArgs(
140 mode='w', mapper=dpTest.EmptyTestMapper,
141 root=repoADir))
142 # create a symlink from 'b' to a temporary directory
143 tmpDir = tempfile.mkdtemp()
144 os.symlink(tmpDir, repoBDir)
145 # create an output repo at 'b' with the input repo 'a'
146 butler = dp.Butler(inputs=repoADir, outputs=repoBDir)
147 self.assertIsInstance(butler, dp.Butler)
148 # verify a repoCfg in the tmp dir with a proper parent path from tmp
149 # to 'a' (not from 'b' to 'a')
150 cfg = dp.PosixStorage.getRepositoryCfg(tmpDir)
151 # verify that the parent link gotten via the symlink target is a path
152 # to A
153 self.assertEqual(repoADir, cfg.parents[0])
154 cfg = dp.PosixStorage.getRepositoryCfg(repoBDir)
155 # also verify that the parent gotten via the symlink is a path to A
156 self.assertEqual(repoADir, cfg.parents[0])
158 def testStorageRepoCfgCache(self):
159 """Tests that when a cfg is gotten from storage it is cached."""
160 butler = dp.Butler(outputs=dp.RepositoryArgs(mode='w',
161 mapper=dpTest.EmptyTestMapper,
162 root=os.path.join(self.testDir, 'a')))
163 del butler
164 storage = dp.Storage()
165 self.assertEqual(0, len(storage.repositoryCfgs))
166 cfg = storage.getRepositoryCfg(os.path.join(self.testDir, 'a'))
167 self.assertEqual(cfg, storage.repositoryCfgs[os.path.join(self.testDir, 'a')])
170class TestNestedCfg(unittest.TestCase):
172 def setUp(self):
173 self.testDir = tempfile.mkdtemp(dir=ROOT,
174 prefix="TestNestedCfg-")
176 def tearDown(self):
177 if os.path.exists(self.testDir):
178 shutil.rmtree(self.testDir)
180 def test(self):
181 parentCfg = dp.RepositoryCfg(root=os.path.join(self.testDir, 'parent'),
182 mapper='lsst.daf.persistence.SomeMapper',
183 mapperArgs={},
184 parents=None,
185 policy=None)
186 cfg = dp.RepositoryCfg(root=self.testDir,
187 mapper='lsst.daf.persistence.SomeMapper',
188 mapperArgs={},
189 parents=[parentCfg],
190 policy=None)
191 dp.PosixStorage.putRepositoryCfg(cfg)
192 reloadedCfg = dp.PosixStorage.getRepositoryCfg(self.testDir)
193 self.assertEqual(cfg, reloadedCfg)
194 self.assertEqual(cfg.parents[0], parentCfg)
197# "fake" repository version 0
198class RepositoryCfg(yaml.YAMLObject):
199 yaml_tag = u"!RepositoryCfg_v0"
201 def __init__(self, mapper, mapperArgs):
202 self._mapper = mapper
203 self._mapperArgs = mapperArgs
205 @staticmethod
206 def v0Constructor(loader, node):
207 d = loader.construct_mapping(node)
208 return dp.RepositoryCfg(root=d['_root'], mapper=None, mapperArgs=None, parents=None, policy=None)
211loaderList = [yaml.Loader, ]
212try:
213 loaderList.append(yaml.UnsafeLoader)
214except AttributeError:
215 pass
217for loader in loaderList:
218 yaml.add_constructor(u"!RepositoryCfg_v0", RepositoryCfg.v0Constructor, Loader=loader)
221class TestCfgFileVersion(unittest.TestCase):
222 """Proof-of-concept test case for a fictitious version 0 of a persisted repository cfg.
223 """
225 def setUp(self):
226 self.testDir = tempfile.mkdtemp(dir=ROOT,
227 prefix="TestCfgFileVersion-")
229 def tearDown(self):
230 if os.path.exists(self.testDir):
231 shutil.rmtree(self.testDir)
233 def test(self):
234 with open(os.path.join(self.testDir, 'repositoryCfg.yaml'), 'w') as f:
235 f.write("""!RepositoryCfg_v0
236 _root: 'foo/bar'""")
237 cfg = dp.PosixStorage.getRepositoryCfg(self.testDir)
238 self.assertIsInstance(cfg, dp.RepositoryCfg)
241class TestExtendParents(unittest.TestCase):
242 """Test for the RepositoryCfg.extendParents function.
243 """
245 def test(self):
246 cfg = dp.RepositoryCfg(root='.', mapper='my.mapper.class', mapperArgs=None,
247 parents=['bar', 'baz'], policy=None)
248 cfg.extendParents(['bar', 'baz', 'qux'])
249 with self.assertRaises(dp.ParentsMismatch):
250 cfg.extendParents(['bar', 'baz', 'corge'])
252 # todo include a repositoryCfg in parents lists in this test
255class TestMapperArgsNone(unittest.TestCase):
256 """Tests that the RepositoryCfg.mapperArgs is converted to a dict if None is passed in.
257 """
258 def test(self):
259 cfg = dp.RepositoryCfg(root='foo', mapper='foo', mapperArgs=None, parents=None, policy=None)
260 self.assertIsInstance(cfg.mapperArgs, dict)
263class MemoryTester(lsst.utils.tests.MemoryTestCase):
264 pass
267def setup_module(module):
268 lsst.utils.tests.init()
271if __name__ == '__main__': 271 ↛ 272line 271 didn't jump to line 272, because the condition on line 271 was never true
272 lsst.utils.tests.init()
273 unittest.main()