Coverage for tests/test_location.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
23import os.path
24import posixpath
26from lsst.daf.butler import LocationFactory, ButlerURI
27from lsst.daf.butler.core.location import os2posix, posix2os
30class LocationTestCase(unittest.TestCase):
31 """Tests for Location within datastore
32 """
34 def testButlerUri(self):
35 """Tests whether ButlerURI instantiates correctly given different
36 arguments.
37 """
38 # Root to use for relative paths
39 testRoot = "/tmp/"
41 # uriStrings is a list of tuples containing test string, forceAbsolute,
42 # forceDirectory as arguments to ButlerURI and scheme, netloc and path
43 # as expected attributes. Test asserts constructed equals to expected.
44 # 1) no determinable schemes (ensures schema and netloc are not set)
45 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
46 uriStrings = [
47 ("relative/file.ext", True, False, "", "", osRelFilePath),
48 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
49 ("test/../relative/file.ext", True, False, "", "", osRelFilePath),
50 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
51 ("relative/dir", False, True, "", "", "relative/dir/")
52 ]
53 # 2) implicit file scheme, tests absolute file and directory paths
54 uriStrings.extend((
55 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'),
56 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
57 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
58 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
59 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
60 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/"))
61 ))
62 # 3) explicit file scheme, absolute and relative file and directory URI
63 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
64 uriStrings.extend((
65 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
66 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
67 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
68 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/")
69 ))
70 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
71 uriStrings.extend((
72 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
73 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
74 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3",
75 "bucketname", "/rootDir/relative/file.ext")
76 ))
78 for uriInfo in uriStrings:
79 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1],
80 forceDirectory=uriInfo[2])
81 with self.subTest(uri=uriInfo[0]):
82 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
83 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
84 self.assertEqual(uri.path, uriInfo[5], "test path")
86 # test root becomes abspath(".") when not specified, note specific
87 # file:// scheme case
88 uriStrings = (
89 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
90 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
91 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"),
92 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext"))
93 )
95 for uriInfo in uriStrings:
96 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
97 with self.subTest(uri=uriInfo[0]):
98 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
99 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
100 self.assertEqual(uri.path, uriInfo[5], "test path")
102 # File replacement
103 uriStrings = (
104 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
105 ("relative/", "newfile.fits", "relative/newfile.fits"),
106 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
107 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
108 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml")
109 )
111 for uriInfo in uriStrings:
112 uri = ButlerURI(uriInfo[0], forceAbsolute=False)
113 uri.updateFile(uriInfo[1])
114 with self.subTest(uri=uriInfo[0]):
115 self.assertEqual(uri.path, uriInfo[2])
117 # Copy constructor
118 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True)
119 uri2 = ButlerURI(uri)
120 self.assertEqual(uri, uri2)
121 uri = ButlerURI("file://amazon/datastore/file.txt")
122 uri2 = ButlerURI(uri)
123 self.assertEqual(uri, uri2)
125 def testFileLocation(self):
126 root = os.path.abspath(os.path.curdir)
127 factory = LocationFactory(root)
128 print(f"Factory created: {factory}")
130 pathInStore = "relative/path/file.ext"
131 loc1 = factory.fromPath(pathInStore)
133 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
134 self.assertEqual(loc1.pathInStore, pathInStore)
135 self.assertTrue(loc1.uri.startswith("file:///"))
136 self.assertTrue(loc1.uri.endswith("file.ext"))
137 loc1.updateExtension("fits")
138 self.assertTrue(loc1.uri.endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}")
139 loc1.updateExtension("fits.gz")
140 self.assertTrue(loc1.uri.endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}")
141 loc1.updateExtension(".jpeg")
142 self.assertTrue(loc1.uri.endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}")
143 loc1.updateExtension(None)
144 self.assertTrue(loc1.uri.endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}")
145 loc1.updateExtension("")
146 self.assertTrue(loc1.uri.endswith("file"), f"Checking no extension in {loc1.uri}")
148 def testRelativeRoot(self):
149 root = os.path.abspath(os.path.curdir)
150 factory = LocationFactory(os.path.curdir)
151 print(f"Factory created: {factory}")
153 pathInStore = "relative/path/file.ext"
154 loc1 = factory.fromPath(pathInStore)
156 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
157 self.assertEqual(loc1.pathInStore, pathInStore)
158 self.assertTrue(loc1.uri.startswith("file:///"))
160 def testHttpLocation(self):
161 root = "https://www.lsst.org/butler/datastore"
162 factory = LocationFactory(root)
163 print(f"Factory created: {factory}")
165 pathInStore = "relative/path/file.ext"
166 loc1 = factory.fromPath(pathInStore)
168 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
169 self.assertEqual(loc1.pathInStore, pathInStore)
170 self.assertTrue(loc1.uri.startswith("https://"))
171 self.assertTrue(loc1.uri.endswith("file.ext"))
172 loc1.updateExtension("fits")
173 self.assertTrue(loc1.uri.endswith("file.fits"))
175 def testPosix2OS(self):
176 """Test round tripping of the posix to os.path conversion helpers."""
177 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
178 for p in testPaths:
179 with self.subTest(path=p):
180 self.assertEqual(os2posix(posix2os(p)), p)
182 def testSplit(self):
183 """Tests split functionality."""
184 testRoot = "/tmp/"
186 testPaths = ("/absolute/file.ext", "/absolute/",
187 "file:///absolute/file.ext", "file:///absolute/",
188 "s3://bucket/root/file.ext", "s3://bucket/root/",
189 "relative/file.ext", "relative/")
191 osRelExpected = os.path.join(testRoot, "relative")
192 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
193 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
194 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""),
195 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", ""))
197 for p, e in zip(testPaths, expected):
198 with self.subTest(path=p):
199 uri = ButlerURI(p, testRoot)
200 head, tail = uri.split()
201 self.assertEqual((head.geturl(), tail), e)
203 # explicit file scheme should force posixpath, check os.path is ignored
204 posixRelFilePath = posixpath.join(testRoot, "relative")
205 uri = ButlerURI("file:relative/file.ext", testRoot)
206 head, tail = uri.split()
207 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
209 # check head can be empty
210 curDir = os.path.abspath(os.path.curdir) + os.sep
211 uri = ButlerURI("file.ext", forceAbsolute=False)
212 head, tail = uri.split()
213 self.assertEqual((head.geturl(), tail), (curDir, "file.ext"))
215 # ensure empty path is not a problem and conforms to os.path.split
216 uri = ButlerURI("", forceAbsolute=False)
217 head, tail = uri.split()
218 self.assertEqual((head.geturl(), tail), (curDir, ""))
220 uri = ButlerURI(".", forceAbsolute=False)
221 head, tail = uri.split()
222 self.assertEqual((head.geturl(), tail), (curDir, "."))
225if __name__ == "__main__": 225 ↛ 226line 225 didn't jump to line 226, because the condition on line 225 was never true
226 unittest.main()