Coverage for tests/test_location.py: 10%
191 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:04 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:04 -0700
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import os.path
14import pathlib
15import pickle
16import posixpath
17import unittest
19from lsst.resources import ResourcePath
20from lsst.resources.location import Location, LocationFactory
21from lsst.resources.utils import os2posix, posix2os
24class LocationTestCase(unittest.TestCase):
25 """Tests for Location within datastore."""
27 def testResourcePath(self):
28 """Tests whether ResourcePath instantiates correctly given different
29 arguments.
30 """
31 # Root to use for relative paths
32 testRoot = "/tmp/"
34 # uriStrings is a list of tuples containing test string, forceAbsolute,
35 # forceDirectory as arguments to ResourcePath and scheme, netloc and
36 # path as expected attributes. Test asserts constructed equals to
37 # expected.
38 # 1) no determinable schemes (ensures schema and netloc are not set)
39 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
40 uriStrings = [
41 ("relative/file.ext", True, False, "file", "", osRelFilePath),
42 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
43 ("test/../relative/file.ext", True, False, "file", "", osRelFilePath),
44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
45 ("relative/dir", False, True, "", "", "relative/dir/"),
46 ]
47 # 2) implicit file scheme, tests absolute file and directory paths
48 uriStrings.extend(
49 (
50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")),
56 )
57 )
58 # 3) explicit file scheme, absolute and relative file and directory URI
59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
60 uriStrings.extend(
61 (
62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"),
66 )
67 )
68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
69 uriStrings.extend(
70 (
71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
73 (
74 "s3://bucketname/rootDir/relative/file.ext",
75 True,
76 False,
77 "s3",
78 "bucketname",
79 "/rootDir/relative/file.ext",
80 ),
81 )
82 )
83 # 5) HTTPS scheme
84 uriStrings.extend(
85 (
86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
88 (
89 "https://www.lsst.org/rootDir/relative/file.ext",
90 True,
91 False,
92 "https",
93 "www.lsst.org",
94 "/rootDir/relative/file.ext",
95 ),
96 )
97 )
99 for uriInfo in uriStrings:
100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
101 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
104 self.assertEqual(uri.path, uriInfo[5], "test path")
106 # test root becomes abspath(".") when not specified, note specific
107 # file:// scheme case
108 uriStrings = (
109 # URI, forceAbsolute, forceDirectory, scheme, netloc, path
110 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
111 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
112 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"),
113 ("relative/file.ext", True, False, "file", "", os.path.abspath("relative/file.ext")),
114 )
116 for uriInfo in uriStrings:
117 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
118 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
119 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
120 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
121 # Use ospath here to ensure that we have unquoted any
122 # special characters in the parent directories.
123 self.assertEqual(uri.ospath, uriInfo[5], "test path")
125 # File replacement
126 uriStrings = (
127 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
128 ("relative/", "newfile.fits", "relative/newfile.fits"),
129 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
130 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
131 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"),
132 )
134 for uriInfo in uriStrings:
135 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
136 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
137 self.assertEqual(uri.path, uriInfo[2])
139 # Check that schemeless can become file scheme.
140 schemeless = ResourcePath("relative/path.ext")
141 filescheme = ResourcePath("/absolute/path.ext")
142 self.assertEqual(schemeless.scheme, "file")
143 self.assertEqual(filescheme.scheme, "file")
144 self.assertEqual(type(schemeless), type(filescheme))
146 # Copy constructor
147 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True)
148 uri2 = ResourcePath(uri)
149 self.assertEqual(uri, uri2)
150 uri = ResourcePath("file://amazon/datastore/file.txt")
151 uri2 = ResourcePath(uri)
152 self.assertEqual(uri, uri2)
154 # Copy constructor using subclass
155 uri3 = type(uri)(uri)
156 self.assertEqual(type(uri), type(uri3))
158 # Explicit copy
159 uri4 = copy.copy(uri3)
160 self.assertEqual(uri4, uri3)
161 uri4 = copy.deepcopy(uri3)
162 self.assertEqual(uri4, uri3)
164 def testUriRoot(self):
165 osPathRoot = pathlib.Path(__file__).absolute().root
166 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com")
167 for uri_str in rootUris:
168 uri = ResourcePath(uri_str, forceDirectory=True)
169 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}")
170 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI")
172 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c")
173 uriStrings = (
174 ("file://localhost/file.ext", "file.ext"),
175 (exampleLocalFile, os.path.join("a", "b", "c")),
176 ("s3://bucket/path/file.ext", "path/file.ext"),
177 ("https://host.com/a/b/c.d", "a/b/c.d"),
178 )
180 for uri_str, result in uriStrings:
181 uri = ResourcePath(uri_str)
182 self.assertEqual(uri.relativeToPathRoot, result)
184 def testUriJoin(self):
185 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False)
186 uri2 = uri.join("e/f/g.txt")
187 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
189 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False)
190 uri2 = uri.join("e/f/g.txt")
191 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
193 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True)
194 uri2 = uri.join("e/f/g.txt")
195 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
197 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True)
198 uri2 = uri.join("newpath/newfile.txt")
199 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
201 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt")
202 uri2 = uri.join("newpath/newfile.txt")
203 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
205 def testResourcePathSerialization(self):
206 """Test that we can pickle and yaml"""
207 uri = ResourcePath("a/b/c/d")
208 uri2 = pickle.loads(pickle.dumps(uri))
209 self.assertEqual(uri, uri2)
210 self.assertFalse(uri2.dirLike)
212 uri = ResourcePath("a/b/c/d", forceDirectory=True)
213 uri2 = pickle.loads(pickle.dumps(uri))
214 self.assertEqual(uri, uri2)
215 self.assertTrue(uri2.dirLike)
217 def testUriExtensions(self):
218 """Test extension extraction."""
220 files = (
221 ("file.fits.gz", ".fits.gz"),
222 ("file.fits", ".fits"),
223 ("file.fits.xz", ".fits.xz"),
224 ("file.fits.tar", ".tar"),
225 ("file", ""),
226 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"),
227 ("flat_i_sim_1.4_blah.txt", ".txt"),
228 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"),
229 ("flat_i_sim_1.4_blah.fits.txt", ".txt"),
230 ("s3://bucket/c/a.b/", ""),
231 ("s3://bucket/c/a.b", ".b"),
232 ("file://localhost/c/a.b.gz", ".b.gz"),
233 )
235 for file, expected in files:
236 test_string = file
237 if ":" not in test_string:
238 test_string = f"a/b/{test_string}"
239 uri = ResourcePath(test_string)
240 self.assertEqual(uri.getExtension(), expected)
242 def testFileLocation(self):
243 root = os.path.abspath(os.path.curdir)
244 factory = LocationFactory(root)
245 print(f"Factory created: {factory}")
247 pathInStore = "relative/path/file.ext"
248 loc1 = factory.fromPath(pathInStore)
250 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
251 self.assertEqual(loc1.pathInStore.path, pathInStore)
252 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
253 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
254 loc1.updateExtension("fits")
255 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}")
256 loc1.updateExtension("fits.gz")
257 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
258 self.assertTrue(
259 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}"
260 )
261 self.assertEqual(loc1.getExtension(), ".fits.gz")
262 loc1.updateExtension(".jpeg")
263 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}")
264 loc1.updateExtension(None)
265 self.assertTrue(
266 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}"
267 )
268 loc1.updateExtension("")
269 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
270 self.assertEqual(loc1.getExtension(), "")
272 loc2 = factory.fromPath(pathInStore)
273 loc3 = factory.fromPath(pathInStore)
274 self.assertEqual(loc2, loc3)
276 def testAbsoluteLocations(self):
277 """Using a pathInStore that refers to absolute URI."""
278 loc = Location(None, "file:///something.txt")
279 self.assertEqual(loc.pathInStore.path, "/something.txt")
280 self.assertEqual(str(loc.uri), "file:///something.txt")
282 with self.assertRaises(ValueError):
283 Location(None, "relative.txt")
285 def testRelativeRoot(self):
286 root = os.path.abspath(os.path.curdir)
287 factory = LocationFactory(os.path.curdir)
289 pathInStore = "relative/path/file.ext"
290 loc1 = factory.fromPath(pathInStore)
292 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
293 self.assertEqual(loc1.pathInStore.path, pathInStore)
294 self.assertEqual(loc1.uri.scheme, "file")
296 with self.assertRaises(ValueError):
297 factory.fromPath("../something")
299 def testQuotedRoot(self):
300 """Test we can handle quoted characters."""
301 root = "/a/b/c+1/d"
302 factory = LocationFactory(root)
304 pathInStore = "relative/path/file.ext.gz"
306 for pathInStore in (
307 "relative/path/file.ext.gz",
308 "relative/path+2/file.ext.gz",
309 "relative/path+3/file&.ext.gz",
310 ):
311 loc1 = factory.fromPath(pathInStore)
313 self.assertEqual(loc1.pathInStore.path, pathInStore)
314 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
315 self.assertIn("%", str(loc1.uri))
316 self.assertEqual(loc1.getExtension(), ".ext.gz")
318 def testHttpLocation(self):
319 root = "https://www.lsst.org/butler/datastore"
320 factory = LocationFactory(root)
321 print(f"Factory created: {factory}")
323 pathInStore = "relative/path/file.ext"
324 loc1 = factory.fromPath(pathInStore)
326 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
327 self.assertEqual(loc1.pathInStore.path, pathInStore)
328 self.assertEqual(loc1.uri.scheme, "https")
329 self.assertEqual(loc1.uri.basename(), "file.ext")
330 loc1.updateExtension("fits")
331 self.assertTrue(loc1.uri.basename(), "file.fits")
333 def testPosix2OS(self):
334 """Test round tripping of the posix to os.path conversion helpers."""
335 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
336 for p in testPaths:
337 with self.subTest(path=p):
338 self.assertEqual(os2posix(posix2os(p)), p)
340 def testSplit(self):
341 """Tests split functionality."""
342 testRoot = "/tmp/"
344 testPaths = (
345 "/absolute/file.ext",
346 "/absolute/",
347 "file:///absolute/file.ext",
348 "file:///absolute/",
349 "s3://bucket/root/file.ext",
350 "s3://bucket/root/",
351 "https://www.lsst.org/root/file.ext",
352 "https://www.lsst.org/root/",
353 "relative/file.ext",
354 "relative/",
355 )
357 osRelExpected = os.path.join(testRoot, "relative")
358 expected = (
359 ("file:///absolute/", "file.ext"),
360 ("file:///absolute/", ""),
361 ("file:///absolute/", "file.ext"),
362 ("file:///absolute/", ""),
363 ("s3://bucket/root/", "file.ext"),
364 ("s3://bucket/root/", ""),
365 ("https://www.lsst.org/root/", "file.ext"),
366 ("https://www.lsst.org/root/", ""),
367 (f"file://{osRelExpected}/", "file.ext"),
368 (f"file://{osRelExpected}/", ""),
369 )
371 for p, e in zip(testPaths, expected):
372 with self.subTest(path=p):
373 uri = ResourcePath(p, testRoot)
374 head, tail = uri.split()
375 self.assertEqual((head.geturl(), tail), e)
377 # explicit file scheme should force posixpath, check os.path is ignored
378 posixRelFilePath = posixpath.join(testRoot, "relative")
379 uri = ResourcePath("file:relative/file.ext", testRoot)
380 head, tail = uri.split()
381 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
383 # check head can be empty and we do not get an absolute path back
384 uri = ResourcePath("file.ext", forceAbsolute=False)
385 head, tail = uri.split()
386 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
388 # ensure empty path splits to a directory URL
389 uri = ResourcePath("", forceAbsolute=False)
390 head, tail = uri.split()
391 self.assertEqual((head.geturl(), tail), ("./", ""))
393 uri = ResourcePath(".", forceAbsolute=False)
394 head, tail = uri.split()
395 self.assertEqual((head.geturl(), tail), ("./", ""))
398if __name__ == "__main__":
399 unittest.main()