Coverage for tests/test_location.py: 10%
191 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 11:34 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-30 11:34 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import copy
13import os.path
14import pathlib
15import pickle
16import posixpath
17import unittest
19from lsst.resources import ResourcePath
20from lsst.resources.location import Location, LocationFactory
21from lsst.resources.utils import os2posix, posix2os
24class LocationTestCase(unittest.TestCase):
25 """Tests for Location within datastore."""
27 def testResourcePath(self):
28 """Tests whether ResourcePath instantiates correctly given different
29 arguments.
30 """
31 # Root to use for relative paths
32 testRoot = "/tempdir/"
34 # uriStrings is a list of tuples containing test string, forceAbsolute,
35 # forceDirectory as arguments to ResourcePath and scheme, netloc and
36 # path as expected attributes. Test asserts constructed equals to
37 # expected.
38 # 1) no determinable schemes (ensures schema and netloc are not set)
39 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
40 uriStrings = [
41 ("relative/file.ext", True, False, "file", "", osRelFilePath),
42 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
43 ("test/../relative/file.ext", True, False, "file", "", osRelFilePath),
44 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
45 ("relative/dir", False, True, "", "", "relative/dir/"),
46 ]
47 # 2) implicit file scheme, tests absolute file and directory paths
48 uriStrings.extend(
49 (
50 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
51 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
52 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
53 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
54 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
55 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")),
56 )
57 )
58 # 3) explicit file scheme, absolute and relative file and directory URI
59 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
60 uriStrings.extend(
61 (
62 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
63 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
64 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
65 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"),
66 )
67 )
68 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
69 uriStrings.extend(
70 (
71 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
72 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
73 (
74 "s3://bucketname/rootDir/relative/file.ext",
75 True,
76 False,
77 "s3",
78 "bucketname",
79 "/rootDir/relative/file.ext",
80 ),
81 )
82 )
83 # 5) HTTPS scheme
84 uriStrings.extend(
85 (
86 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
87 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
88 (
89 "https://www.lsst.org/rootDir/relative/file.ext",
90 True,
91 False,
92 "https",
93 "www.lsst.org",
94 "/rootDir/relative/file.ext",
95 ),
96 )
97 )
99 for uriInfo in uriStrings:
100 uri = ResourcePath(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
101 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
102 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
103 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
104 self.assertEqual(uri.path, uriInfo[5], "test path")
106 # test root becomes abspath(".") when not specified, note specific
107 # file:// scheme case
108 uriStrings = (
109 # URI, forceAbsolute, forceDirectory, scheme, netloc, path
110 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
111 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
112 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"),
113 ("relative/file.ext", True, False, "file", "", os.path.abspath("relative/file.ext")),
114 )
116 for uriInfo in uriStrings:
117 uri = ResourcePath(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
118 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
119 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
120 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
121 # Use ospath here to ensure that we have unquoted any
122 # special characters in the parent directories.
123 self.assertEqual(uri.ospath, uriInfo[5], "test path")
125 # File replacement
126 uriStrings = (
127 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
128 ("relative/", "newfile.fits", "relative/newfile.fits"),
129 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
130 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
131 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"),
132 )
134 for uriInfo in uriStrings:
135 uri = ResourcePath(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
136 with self.subTest(in_uri=uriInfo[0], out_uri=uri):
137 self.assertEqual(uri.path, uriInfo[2])
139 # Check that schemeless can become file scheme.
140 schemeless = ResourcePath("relative/path.ext")
141 filescheme = ResourcePath("/absolute/path.ext")
142 self.assertEqual(schemeless.scheme, "file")
143 self.assertEqual(filescheme.scheme, "file")
144 self.assertEqual(type(schemeless), type(filescheme))
146 # Copy constructor
147 uri = ResourcePath("s3://amazon/datastore", forceDirectory=True)
148 uri2 = ResourcePath(uri)
149 self.assertEqual(uri, uri2)
150 uri = ResourcePath("file://amazon/datastore/file.txt")
151 uri2 = ResourcePath(uri)
152 self.assertEqual(uri, uri2)
154 # Copy constructor using subclass
155 uri3 = type(uri)(uri)
156 self.assertEqual(type(uri), type(uri3))
158 # Explicit copy
159 uri4 = copy.copy(uri3)
160 self.assertEqual(uri4, uri3)
161 uri4 = copy.deepcopy(uri3)
162 self.assertEqual(uri4, uri3)
164 def testUriRoot(self):
165 osPathRoot = pathlib.Path(__file__).absolute().root
166 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com")
167 for uri_str in rootUris:
168 uri = ResourcePath(uri_str, forceDirectory=True)
169 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}")
170 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI")
172 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c")
173 uriStrings = (
174 ("file://localhost/file.ext", "file.ext"),
175 (exampleLocalFile, os.path.join("a", "b", "c")),
176 ("s3://bucket/path/file.ext", "path/file.ext"),
177 ("https://host.com/a/b/c.d", "a/b/c.d"),
178 )
180 for uri_str, result in uriStrings:
181 uri = ResourcePath(uri_str)
182 self.assertEqual(uri.relativeToPathRoot, result)
184 def testUriJoin(self):
185 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=False)
186 uri2 = uri.join("e/f/g.txt")
187 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
189 uri = ResourcePath("a/b/c/d/old.txt", forceAbsolute=False)
190 uri2 = uri.join("e/f/g.txt")
191 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
193 uri = ResourcePath("a/b/c/d", forceDirectory=True, forceAbsolute=True)
194 uri2 = uri.join("e/f/g.txt")
195 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
197 uri = ResourcePath("s3://bucket/a/b/c/d", forceDirectory=True)
198 uri2 = uri.join("newpath/newfile.txt")
199 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
201 uri = ResourcePath("s3://bucket/a/b/c/d/old.txt")
202 uri2 = uri.join("newpath/newfile.txt")
203 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
205 def testResourcePathSerialization(self):
206 """Test that we can pickle and yaml"""
207 uri = ResourcePath("a/b/c/d")
208 uri2 = pickle.loads(pickle.dumps(uri))
209 self.assertEqual(uri, uri2)
210 self.assertFalse(uri2.dirLike)
212 uri = ResourcePath("a/b/c/d", forceDirectory=True)
213 uri2 = pickle.loads(pickle.dumps(uri))
214 self.assertEqual(uri, uri2)
215 self.assertTrue(uri2.dirLike)
217 def testUriExtensions(self):
218 """Test extension extraction."""
219 files = (
220 ("file.fits.gz", ".fits.gz"),
221 ("file.fits", ".fits"),
222 ("file.fits.xz", ".fits.xz"),
223 ("file.fits.tar", ".tar"),
224 ("file", ""),
225 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"),
226 ("flat_i_sim_1.4_blah.txt", ".txt"),
227 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"),
228 ("flat_i_sim_1.4_blah.fits.txt", ".txt"),
229 ("s3://bucket/c/a.b/", ""),
230 ("s3://bucket/c/a.b", ".b"),
231 ("file://localhost/c/a.b.gz", ".b.gz"),
232 )
234 for file, expected in files:
235 test_string = file
236 if ":" not in test_string:
237 test_string = f"a/b/{test_string}"
238 uri = ResourcePath(test_string)
239 self.assertEqual(uri.getExtension(), expected)
241 def testFileLocation(self):
242 root = os.path.abspath(os.path.curdir)
243 factory = LocationFactory(root)
244 print(f"Factory created: {factory}")
246 pathInStore = "relative/path/file.ext"
247 loc1 = factory.fromPath(pathInStore)
249 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
250 self.assertEqual(loc1.pathInStore.path, pathInStore)
251 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
252 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
253 loc1.updateExtension("fits")
254 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}")
255 loc1.updateExtension("fits.gz")
256 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
257 self.assertTrue(
258 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}"
259 )
260 self.assertEqual(loc1.getExtension(), ".fits.gz")
261 loc1.updateExtension(".jpeg")
262 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}")
263 loc1.updateExtension(None)
264 self.assertTrue(
265 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}"
266 )
267 loc1.updateExtension("")
268 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
269 self.assertEqual(loc1.getExtension(), "")
271 loc2 = factory.fromPath(pathInStore)
272 loc3 = factory.fromPath(pathInStore)
273 self.assertEqual(loc2, loc3)
275 def testAbsoluteLocations(self):
276 """Using a pathInStore that refers to absolute URI."""
277 loc = Location(None, "file:///something.txt")
278 self.assertEqual(loc.pathInStore.path, "/something.txt")
279 self.assertEqual(str(loc.uri), "file:///something.txt")
281 with self.assertRaises(ValueError):
282 Location(None, "relative.txt")
284 def testRelativeRoot(self):
285 root = os.path.abspath(os.path.curdir)
286 factory = LocationFactory(os.path.curdir)
288 pathInStore = "relative/path/file.ext"
289 loc1 = factory.fromPath(pathInStore)
291 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
292 self.assertEqual(loc1.pathInStore.path, pathInStore)
293 self.assertEqual(loc1.uri.scheme, "file")
295 with self.assertRaises(ValueError):
296 factory.fromPath("../something")
298 def testQuotedRoot(self):
299 """Test we can handle quoted characters."""
300 root = "/a/b/c+1/d"
301 factory = LocationFactory(root)
303 pathInStore = "relative/path/file.ext.gz"
305 for pathInStore in (
306 "relative/path/file.ext.gz",
307 "relative/path+2/file.ext.gz",
308 "relative/path+3/file&.ext.gz",
309 ):
310 loc1 = factory.fromPath(pathInStore)
312 self.assertEqual(loc1.pathInStore.path, pathInStore)
313 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
314 self.assertIn("%", str(loc1.uri))
315 self.assertEqual(loc1.getExtension(), ".ext.gz")
317 def testHttpLocation(self):
318 root = "https://www.lsst.org/butler/datastore"
319 factory = LocationFactory(root)
320 print(f"Factory created: {factory}")
322 pathInStore = "relative/path/file.ext"
323 loc1 = factory.fromPath(pathInStore)
325 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
326 self.assertEqual(loc1.pathInStore.path, pathInStore)
327 self.assertEqual(loc1.uri.scheme, "https")
328 self.assertEqual(loc1.uri.basename(), "file.ext")
329 loc1.updateExtension("fits")
330 self.assertTrue(loc1.uri.basename(), "file.fits")
332 def testPosix2OS(self):
333 """Test round tripping of the posix to os.path conversion helpers."""
334 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
335 for p in testPaths:
336 with self.subTest(path=p):
337 self.assertEqual(os2posix(posix2os(p)), p)
339 def testSplit(self):
340 """Tests split functionality."""
341 testRoot = "/tempdir/"
343 testPaths = (
344 "/absolute/file.ext",
345 "/absolute/",
346 "file:///absolute/file.ext",
347 "file:///absolute/",
348 "s3://bucket/root/file.ext",
349 "s3://bucket/root/",
350 "https://www.lsst.org/root/file.ext",
351 "https://www.lsst.org/root/",
352 "relative/file.ext",
353 "relative/",
354 )
356 osRelExpected = os.path.join(testRoot, "relative")
357 expected = (
358 ("file:///absolute/", "file.ext"),
359 ("file:///absolute/", ""),
360 ("file:///absolute/", "file.ext"),
361 ("file:///absolute/", ""),
362 ("s3://bucket/root/", "file.ext"),
363 ("s3://bucket/root/", ""),
364 ("https://www.lsst.org/root/", "file.ext"),
365 ("https://www.lsst.org/root/", ""),
366 (f"file://{osRelExpected}/", "file.ext"),
367 (f"file://{osRelExpected}/", ""),
368 )
370 for p, e in zip(testPaths, expected, strict=True):
371 with self.subTest(path=p):
372 uri = ResourcePath(p, testRoot)
373 head, tail = uri.split()
374 self.assertEqual((head.geturl(), tail), e)
376 # explicit file scheme should force posixpath, check os.path is ignored
377 posixRelFilePath = posixpath.join(testRoot, "relative")
378 uri = ResourcePath("file:relative/file.ext", testRoot)
379 head, tail = uri.split()
380 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
382 # check head can be empty and we do not get an absolute path back
383 uri = ResourcePath("file.ext", forceAbsolute=False)
384 head, tail = uri.split()
385 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
387 # ensure empty path splits to a directory URL
388 uri = ResourcePath("", forceAbsolute=False)
389 head, tail = uri.split()
390 self.assertEqual((head.geturl(), tail), ("./", ""))
392 uri = ResourcePath(".", forceAbsolute=False)
393 head, tail = uri.split()
394 self.assertEqual((head.geturl(), tail), ("./", ""))
397if __name__ == "__main__":
398 unittest.main()