Coverage for tests/test_location.py: 10%
193 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-16 02:51 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-16 02:51 -0800
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import os.path
14import pathlib
15import pickle
16import posixpath
17import unittest
19from lsst.resources import ResourcePath
20from lsst.resources.location import Location, LocationFactory
21from lsst.resources.utils import os2posix, posix2os
24class LocationTestCase(unittest.TestCase):
25 """Tests for Location within datastore."""
27 def testResourcePath(self):
28 """Tests whether ResourcePath instantiates correctly given different
29 arguments.
30 """
31 # Root to use for relative paths
32 testRoot = "/tmp/"
34 # uriStrings is a list of tuples containing test string, forceAbsolute,
35 # forceDirectory as arguments to ResourcePath and scheme, netloc and
36 # path as expected attributes. Test asserts constructed equals to
37 # expected.
38 # 1) no determinable schemes (ensures schema and netloc are not set)
39 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
40 uriStrings = [
41 ("relative/file.ext", True, False, "", "", osRelFilePath),
42 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
43 ("test/../relative/file.ext", True, False, "", "", osRelFilePath),
44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
45 ("relative/dir", False, True, "", "", "relative/dir/"),
46 ]
47 # 2) implicit file scheme, tests absolute file and directory paths
48 uriStrings.extend(
49 (
50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")),
56 )
57 )
58 # 3) explicit file scheme, absolute and relative file and directory URI
59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
60 uriStrings.extend(
61 (
62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"),
66 )
67 )
68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
69 uriStrings.extend(
70 (
71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
73 (
74 "s3://bucketname/rootDir/relative/file.ext",
75 True,
76 False,
77 "s3",
78 "bucketname",
79 "/rootDir/relative/file.ext",
80 ),
81 )
82 )
83 # 5) HTTPS scheme
84 uriStrings.extend(
85 (
86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
88 (
89 "https://www.lsst.org/rootDir/relative/file.ext",
90 True,
91 False,
92 "https",
93 "www.lsst.org",
94 "/rootDir/relative/file.ext",
95 ),
96 )
97 )
99 for uriInfo in uriStrings:
100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
101 with self.subTest(uri=uriInfo[0]):
102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
104 self.assertEqual(uri.path, uriInfo[5], "test path")
106 # test root becomes abspath(".") when not specified, note specific
107 # file:// scheme case
108 uriStrings = (
109 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
110 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
111 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"),
112 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")),
113 )
115 for uriInfo in uriStrings:
116 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
117 with self.subTest(uri=uriInfo[0]):
118 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
119 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
120 self.assertEqual(uri.path, uriInfo[5], "test path")
122 # File replacement
123 uriStrings = (
124 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
125 ("relative/", "newfile.fits", "relative/newfile.fits"),
126 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
127 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
128 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"),
129 )
131 for uriInfo in uriStrings:
132 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
133 with self.subTest(uri=uriInfo[0]):
134 self.assertEqual(uri.path, uriInfo[2])
136 # Check that schemeless can become file scheme
137 schemeless = ResourcePath("relative/path.ext")
138 filescheme = ResourcePath("/absolute/path.ext")
139 self.assertFalse(schemeless.scheme)
140 self.assertEqual(filescheme.scheme, "file")
141 self.assertNotEqual(type(schemeless), type(filescheme))
143 # Copy constructor
144 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True)
145 uri2 = ResourcePath(uri)
146 self.assertEqual(uri, uri2)
147 uri = ResourcePath("file://amazon/datastore/file.txt")
148 uri2 = ResourcePath(uri)
149 self.assertEqual(uri, uri2)
151 # Copy constructor using subclass
152 uri3 = type(uri)(uri)
153 self.assertEqual(type(uri), type(uri3))
155 # Explicit copy
156 uri4 = copy.copy(uri3)
157 self.assertEqual(uri4, uri3)
158 uri4 = copy.deepcopy(uri3)
159 self.assertEqual(uri4, uri3)
161 def testUriRoot(self):
162 osPathRoot = pathlib.Path(__file__).absolute().root
163 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com")
164 for uri_str in rootUris:
165 uri = ResourcePath(uri_str, forceDirectory=True)
166 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}")
167 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI")
169 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c")
170 uriStrings = (
171 ("file://localhost/file.ext", "file.ext"),
172 (exampleLocalFile, os.path.join("a", "b", "c")),
173 ("s3://bucket/path/file.ext", "path/file.ext"),
174 ("https://host.com/a/b/c.d", "a/b/c.d"),
175 )
177 for uri_str, result in uriStrings:
178 uri = ResourcePath(uri_str)
179 self.assertEqual(uri.relativeToPathRoot, result)
181 def testUriJoin(self):
182 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False)
183 uri2 = uri.join("e/f/g.txt")
184 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
186 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False)
187 uri2 = uri.join("e/f/g.txt")
188 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
190 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True)
191 uri2 = uri.join("e/f/g.txt")
192 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
194 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True)
195 uri2 = uri.join("newpath/newfile.txt")
196 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
198 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt")
199 uri2 = uri.join("newpath/newfile.txt")
200 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
202 def testResourcePathSerialization(self):
203 """Test that we can pickle and yaml"""
204 uri = ResourcePath("a/b/c/d")
205 uri2 = pickle.loads(pickle.dumps(uri))
206 self.assertEqual(uri, uri2)
207 self.assertFalse(uri2.dirLike)
209 uri = ResourcePath("a/b/c/d", forceDirectory=True)
210 uri2 = pickle.loads(pickle.dumps(uri))
211 self.assertEqual(uri, uri2)
212 self.assertTrue(uri2.dirLike)
214 def testUriExtensions(self):
215 """Test extension extraction."""
217 files = (
218 ("file.fits.gz", ".fits.gz"),
219 ("file.fits", ".fits"),
220 ("file.fits.xz", ".fits.xz"),
221 ("file.fits.tar", ".tar"),
222 ("file", ""),
223 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"),
224 ("flat_i_sim_1.4_blah.txt", ".txt"),
225 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"),
226 ("flat_i_sim_1.4_blah.fits.txt", ".txt"),
227 ("s3://bucket/c/a.b/", ""),
228 ("s3://bucket/c/a.b", ".b"),
229 ("file://localhost/c/a.b.gz", ".b.gz"),
230 )
232 for file, expected in files:
233 test_string = file
234 if ":" not in test_string:
235 test_string = f"a/b/{test_string}"
236 uri = ResourcePath(test_string)
237 self.assertEqual(uri.getExtension(), expected)
239 def testFileLocation(self):
240 root = os.path.abspath(os.path.curdir)
241 factory = LocationFactory(root)
242 print(f"Factory created: {factory}")
244 pathInStore = "relative/path/file.ext"
245 loc1 = factory.fromPath(pathInStore)
247 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
248 self.assertEqual(loc1.pathInStore.path, pathInStore)
249 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
250 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
251 loc1.updateExtension("fits")
252 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}")
253 loc1.updateExtension("fits.gz")
254 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
255 self.assertTrue(
256 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}"
257 )
258 self.assertEqual(loc1.getExtension(), ".fits.gz")
259 loc1.updateExtension(".jpeg")
260 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}")
261 loc1.updateExtension(None)
262 self.assertTrue(
263 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}"
264 )
265 loc1.updateExtension("")
266 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
267 self.assertEqual(loc1.getExtension(), "")
269 loc2 = factory.fromPath(pathInStore)
270 loc3 = factory.fromPath(pathInStore)
271 self.assertEqual(loc2, loc3)
273 def testAbsoluteLocations(self):
274 """Using a pathInStore that refers to absolute URI."""
275 loc = Location(None, "file:///something.txt")
276 self.assertEqual(loc.pathInStore.path, "/something.txt")
277 self.assertEqual(str(loc.uri), "file:///something.txt")
279 with self.assertRaises(ValueError):
280 Location(None, "relative.txt")
282 def testRelativeRoot(self):
283 root = os.path.abspath(os.path.curdir)
284 factory = LocationFactory(os.path.curdir)
286 pathInStore = "relative/path/file.ext"
287 loc1 = factory.fromPath(pathInStore)
289 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
290 self.assertEqual(loc1.pathInStore.path, pathInStore)
291 self.assertEqual(loc1.uri.scheme, "file")
293 with self.assertRaises(ValueError):
294 factory.fromPath("../something")
296 def testQuotedRoot(self):
297 """Test we can handle quoted characters."""
298 root = "/a/b/c+1/d"
299 factory = LocationFactory(root)
301 pathInStore = "relative/path/file.ext.gz"
303 for pathInStore in (
304 "relative/path/file.ext.gz",
305 "relative/path+2/file.ext.gz",
306 "relative/path+3/file&.ext.gz",
307 ):
308 loc1 = factory.fromPath(pathInStore)
310 self.assertEqual(loc1.pathInStore.path, pathInStore)
311 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
312 self.assertIn("%", str(loc1.uri))
313 self.assertEqual(loc1.getExtension(), ".ext.gz")
315 def testHttpLocation(self):
316 root = "https://www.lsst.org/butler/datastore"
317 factory = LocationFactory(root)
318 print(f"Factory created: {factory}")
320 pathInStore = "relative/path/file.ext"
321 loc1 = factory.fromPath(pathInStore)
323 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
324 self.assertEqual(loc1.pathInStore.path, pathInStore)
325 self.assertEqual(loc1.uri.scheme, "https")
326 self.assertEqual(loc1.uri.basename(), "file.ext")
327 loc1.updateExtension("fits")
328 self.assertTrue(loc1.uri.basename(), "file.fits")
330 def testPosix2OS(self):
331 """Test round tripping of the posix to os.path conversion helpers."""
332 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
333 for p in testPaths:
334 with self.subTest(path=p):
335 self.assertEqual(os2posix(posix2os(p)), p)
337 def testSplit(self):
338 """Tests split functionality."""
339 testRoot = "/tmp/"
341 testPaths = (
342 "/absolute/file.ext",
343 "/absolute/",
344 "file:///absolute/file.ext",
345 "file:///absolute/",
346 "s3://bucket/root/file.ext",
347 "s3://bucket/root/",
348 "https://www.lsst.org/root/file.ext",
349 "https://www.lsst.org/root/",
350 "relative/file.ext",
351 "relative/",
352 )
354 osRelExpected = os.path.join(testRoot, "relative")
355 expected = (
356 ("file:///absolute/", "file.ext"),
357 ("file:///absolute/", ""),
358 ("file:///absolute/", "file.ext"),
359 ("file:///absolute/", ""),
360 ("s3://bucket/root/", "file.ext"),
361 ("s3://bucket/root/", ""),
362 ("https://www.lsst.org/root/", "file.ext"),
363 ("https://www.lsst.org/root/", ""),
364 (f"file://{osRelExpected}/", "file.ext"),
365 (f"file://{osRelExpected}/", ""),
366 )
368 for p, e in zip(testPaths, expected):
369 with self.subTest(path=p):
370 uri = ResourcePath(p, testRoot)
371 head, tail = uri.split()
372 self.assertEqual((head.geturl(), tail), e)
374 # explicit file scheme should force posixpath, check os.path is ignored
375 posixRelFilePath = posixpath.join(testRoot, "relative")
376 uri = ResourcePath("file:relative/file.ext", testRoot)
377 head, tail = uri.split()
378 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
380 # check head can be empty and we do not get an absolute path back
381 uri = ResourcePath("file.ext", forceAbsolute=False)
382 head, tail = uri.split()
383 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
385 # ensure empty path splits to a directory URL
386 uri = ResourcePath("", forceAbsolute=False)
387 head, tail = uri.split()
388 self.assertEqual((head.geturl(), tail), ("./", ""))
390 uri = ResourcePath(".", forceAbsolute=False)
391 head, tail = uri.split()
392 self.assertEqual((head.geturl(), tail), ("./", ""))
395if __name__ == "__main__": 395 ↛ 396line 395 didn't jump to line 396, because the condition on line 395 was never true
396 unittest.main()