Coverage for tests/test_location.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import copy
23import unittest
24import os.path
25import posixpath
26import pickle
28from lsst.daf.butler import LocationFactory, ButlerURI
29from lsst.daf.butler.core._butlerUri import os2posix, posix2os
32class LocationTestCase(unittest.TestCase):
33 """Tests for Location within datastore
34 """
36 def testButlerUri(self):
37 """Tests whether ButlerURI instantiates correctly given different
38 arguments.
39 """
40 # Root to use for relative paths
41 testRoot = "/tmp/"
43 # uriStrings is a list of tuples containing test string, forceAbsolute,
44 # forceDirectory as arguments to ButlerURI and scheme, netloc and path
45 # as expected attributes. Test asserts constructed equals to expected.
46 # 1) no determinable schemes (ensures schema and netloc are not set)
47 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
48 uriStrings = [
49 ("relative/file.ext", True, False, "", "", osRelFilePath),
50 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
51 ("test/../relative/file.ext", True, False, "", "", osRelFilePath),
52 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
53 ("relative/dir", False, True, "", "", "relative/dir/")
54 ]
55 # 2) implicit file scheme, tests absolute file and directory paths
56 uriStrings.extend((
57 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'),
58 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
59 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
60 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
61 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
62 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/"))
63 ))
64 # 3) explicit file scheme, absolute and relative file and directory URI
65 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
66 uriStrings.extend((
67 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
68 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
69 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
70 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/")
71 ))
72 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
73 uriStrings.extend((
74 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
75 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
76 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3",
77 "bucketname", "/rootDir/relative/file.ext")
78 ))
80 for uriInfo in uriStrings:
81 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1],
82 forceDirectory=uriInfo[2])
83 with self.subTest(uri=uriInfo[0]):
84 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
85 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
86 self.assertEqual(uri.path, uriInfo[5], "test path")
88 # test root becomes abspath(".") when not specified, note specific
89 # file:// scheme case
90 uriStrings = (
91 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
92 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
93 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"),
94 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext"))
95 )
97 for uriInfo in uriStrings:
98 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
99 with self.subTest(uri=uriInfo[0]):
100 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
101 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
102 self.assertEqual(uri.path, uriInfo[5], "test path")
104 # File replacement
105 uriStrings = (
106 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
107 ("relative/", "newfile.fits", "relative/newfile.fits"),
108 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
109 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
110 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml")
111 )
113 for uriInfo in uriStrings:
114 uri = ButlerURI(uriInfo[0], forceAbsolute=False)
115 uri.updateFile(uriInfo[1])
116 with self.subTest(uri=uriInfo[0]):
117 self.assertEqual(uri.path, uriInfo[2])
119 # Check that schemeless can become file scheme
120 schemeless = ButlerURI("relative/path.ext")
121 filescheme = ButlerURI("/absolute/path.ext")
122 self.assertFalse(schemeless.scheme)
123 self.assertEqual(filescheme.scheme, "file")
124 self.assertNotEqual(type(schemeless), type(filescheme))
126 # Copy constructor
127 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True)
128 uri2 = ButlerURI(uri)
129 self.assertEqual(uri, uri2)
130 uri = ButlerURI("file://amazon/datastore/file.txt")
131 uri2 = ButlerURI(uri)
132 self.assertEqual(uri, uri2)
134 # Copy constructor using subclass
135 uri3 = type(uri)(uri)
136 self.assertEqual(type(uri), type(uri3))
138 # Explicit copy
139 uri4 = copy.copy(uri3)
140 self.assertEqual(uri4, uri3)
141 uri4 = copy.deepcopy(uri3)
142 self.assertEqual(uri4, uri3)
144 def testUriJoin(self):
145 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False)
146 uri2 = uri.join("e/f/g.txt")
147 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
149 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False)
150 uri2 = uri.join("e/f/g.txt")
151 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
153 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True)
154 uri2 = uri.join("e/f/g.txt")
155 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
157 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True)
158 uri2 = uri.join("newpath/newfile.txt")
159 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
161 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt")
162 uri2 = uri.join("newpath/newfile.txt")
163 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
165 def testButlerUriSerialization(self):
166 """Test that we can pickle and yaml"""
167 uri = ButlerURI("a/b/c/d")
168 uri2 = pickle.loads(pickle.dumps(uri))
169 self.assertEqual(uri, uri2)
170 self.assertFalse(uri2.dirLike)
172 uri = ButlerURI("a/b/c/d", forceDirectory=True)
173 uri2 = pickle.loads(pickle.dumps(uri))
174 self.assertEqual(uri, uri2)
175 self.assertTrue(uri2.dirLike)
177 def testFileLocation(self):
178 root = os.path.abspath(os.path.curdir)
179 factory = LocationFactory(root)
180 print(f"Factory created: {factory}")
182 pathInStore = "relative/path/file.ext"
183 loc1 = factory.fromPath(pathInStore)
185 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
186 self.assertEqual(loc1.pathInStore, pathInStore)
187 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
188 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
189 loc1.updateExtension("fits")
190 self.assertTrue(loc1.uri.geturl().endswith("file.fits"),
191 f"Checking 'fits' extension in {loc1.uri}")
192 loc1.updateExtension("fits.gz")
193 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
194 self.assertTrue(loc1.uri.geturl().endswith("file.fits.gz"),
195 f"Checking 'fits.gz' extension in {loc1.uri}")
196 self.assertEqual(loc1.getExtension(), ".fits.gz")
197 loc1.updateExtension(".jpeg")
198 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"),
199 f"Checking 'jpeg' extension in {loc1.uri}")
200 loc1.updateExtension(None)
201 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"),
202 f"Checking unchanged extension in {loc1.uri}")
203 loc1.updateExtension("")
204 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
205 self.assertEqual(loc1.getExtension(), "")
207 def testRelativeRoot(self):
208 root = os.path.abspath(os.path.curdir)
209 factory = LocationFactory(os.path.curdir)
211 pathInStore = "relative/path/file.ext"
212 loc1 = factory.fromPath(pathInStore)
214 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
215 self.assertEqual(loc1.pathInStore, pathInStore)
216 self.assertEqual(loc1.uri.scheme, "file")
218 def testQuotedRoot(self):
219 """Test we can handle quoted characters."""
220 root = "/a/b/c+1/d"
221 factory = LocationFactory(root)
223 pathInStore = "relative/path/file.ext.gz"
225 for pathInStore in ("relative/path/file.ext.gz",
226 "relative/path+2/file.ext.gz",
227 "relative/path+3/file#.ext.gz"):
228 loc1 = factory.fromPath(pathInStore)
230 self.assertEqual(loc1.pathInStore, pathInStore)
231 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
232 self.assertIn("%", str(loc1.uri))
233 self.assertEqual(loc1.getExtension(), ".ext.gz")
235 def testHttpLocation(self):
236 root = "https://www.lsst.org/butler/datastore"
237 factory = LocationFactory(root)
238 print(f"Factory created: {factory}")
240 pathInStore = "relative/path/file.ext"
241 loc1 = factory.fromPath(pathInStore)
243 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
244 self.assertEqual(loc1.pathInStore, pathInStore)
245 self.assertEqual(loc1.uri.scheme, "https")
246 self.assertEqual(loc1.uri.basename(), "file.ext")
247 loc1.updateExtension("fits")
248 self.assertTrue(loc1.uri.basename(), "file.fits")
250 def testPosix2OS(self):
251 """Test round tripping of the posix to os.path conversion helpers."""
252 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
253 for p in testPaths:
254 with self.subTest(path=p):
255 self.assertEqual(os2posix(posix2os(p)), p)
257 def testSplit(self):
258 """Tests split functionality."""
259 testRoot = "/tmp/"
261 testPaths = ("/absolute/file.ext", "/absolute/",
262 "file:///absolute/file.ext", "file:///absolute/",
263 "s3://bucket/root/file.ext", "s3://bucket/root/",
264 "relative/file.ext", "relative/")
266 osRelExpected = os.path.join(testRoot, "relative")
267 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
268 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
269 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""),
270 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", ""))
272 for p, e in zip(testPaths, expected):
273 with self.subTest(path=p):
274 uri = ButlerURI(p, testRoot)
275 head, tail = uri.split()
276 self.assertEqual((head.geturl(), tail), e)
278 # explicit file scheme should force posixpath, check os.path is ignored
279 posixRelFilePath = posixpath.join(testRoot, "relative")
280 uri = ButlerURI("file:relative/file.ext", testRoot)
281 head, tail = uri.split()
282 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
284 # check head can be empty and we do not get an absolute path back
285 uri = ButlerURI("file.ext", forceAbsolute=False)
286 head, tail = uri.split()
287 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
289 # ensure empty path splits to a directory URL
290 uri = ButlerURI("", forceAbsolute=False)
291 head, tail = uri.split()
292 self.assertEqual((head.geturl(), tail), ("./", ""))
294 uri = ButlerURI(".", forceAbsolute=False)
295 head, tail = uri.split()
296 self.assertEqual((head.geturl(), tail), ("./", ""))
299if __name__ == "__main__": 299 ↛ 300line 299 didn't jump to line 300, because the condition on line 299 was never true
300 unittest.main()