Coverage for tests/test_location.py: 12%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import copy
23import os.path
24import pathlib
25import pickle
26import posixpath
27import unittest
29from lsst.daf.butler import ButlerURI, Location, LocationFactory
30from lsst.resources.utils import os2posix, posix2os
33class LocationTestCase(unittest.TestCase):
34 """Tests for Location within datastore"""
36 def testButlerUri(self):
37 """Tests whether ButlerURI instantiates correctly given different
38 arguments.
39 """
40 # Root to use for relative paths
41 testRoot = "/tmp/"
43 # uriStrings is a list of tuples containing test string, forceAbsolute,
44 # forceDirectory as arguments to ButlerURI and scheme, netloc and path
45 # as expected attributes. Test asserts constructed equals to expected.
46 # 1) no determinable schemes (ensures schema and netloc are not set)
47 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
48 uriStrings = [
49 ("relative/file.ext", True, False, "", "", osRelFilePath),
50 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
51 ("test/../relative/file.ext", True, False, "", "", osRelFilePath),
52 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
53 ("relative/dir", False, True, "", "", "relative/dir/"),
54 ]
55 # 2) implicit file scheme, tests absolute file and directory paths
56 uriStrings.extend(
57 (
58 ("/rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
59 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
60 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
61 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
62 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
63 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/")),
64 )
65 )
66 # 3) explicit file scheme, absolute and relative file and directory URI
67 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
68 uriStrings.extend(
69 (
70 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
71 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
72 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
73 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/"),
74 )
75 )
76 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
77 uriStrings.extend(
78 (
79 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
80 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
81 (
82 "s3://bucketname/rootDir/relative/file.ext",
83 True,
84 False,
85 "s3",
86 "bucketname",
87 "/rootDir/relative/file.ext",
88 ),
89 )
90 )
91 # 5) HTTPS scheme
92 uriStrings.extend(
93 (
94 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
95 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
96 (
97 "https://www.lsst.org/rootDir/relative/file.ext",
98 True,
99 False,
100 "https",
101 "www.lsst.org",
102 "/rootDir/relative/file.ext",
103 ),
104 )
105 )
107 for uriInfo in uriStrings:
108 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
109 with self.subTest(uri=uriInfo[0]):
110 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
111 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
112 self.assertEqual(uri.path, uriInfo[5], "test path")
114 # test root becomes abspath(".") when not specified, note specific
115 # file:// scheme case
116 uriStrings = (
117 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
118 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
119 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir") + "/"),
120 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext")),
121 )
123 for uriInfo in uriStrings:
124 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
125 with self.subTest(uri=uriInfo[0]):
126 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
127 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
128 self.assertEqual(uri.path, uriInfo[5], "test path")
130 # File replacement
131 uriStrings = (
132 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
133 ("relative/", "newfile.fits", "relative/newfile.fits"),
134 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
135 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
136 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml"),
137 )
139 for uriInfo in uriStrings:
140 uri = ButlerURI(uriInfo[0], forceAbsolute=False).updatedFile(uriInfo[1])
141 with self.subTest(uri=uriInfo[0]):
142 self.assertEqual(uri.path, uriInfo[2])
144 # Check that schemeless can become file scheme
145 schemeless = ButlerURI("relative/path.ext")
146 filescheme = ButlerURI("/absolute/path.ext")
147 self.assertFalse(schemeless.scheme)
148 self.assertEqual(filescheme.scheme, "file")
149 self.assertNotEqual(type(schemeless), type(filescheme))
151 # Copy constructor
152 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True)
153 uri2 = ButlerURI(uri)
154 self.assertEqual(uri, uri2)
155 uri = ButlerURI("file://amazon/datastore/file.txt")
156 uri2 = ButlerURI(uri)
157 self.assertEqual(uri, uri2)
159 # Copy constructor using subclass
160 uri3 = type(uri)(uri)
161 self.assertEqual(type(uri), type(uri3))
163 # Explicit copy
164 uri4 = copy.copy(uri3)
165 self.assertEqual(uri4, uri3)
166 uri4 = copy.deepcopy(uri3)
167 self.assertEqual(uri4, uri3)
169 def testUriRoot(self):
170 osPathRoot = pathlib.Path(__file__).absolute().root
171 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com")
172 for uri_str in rootUris:
173 uri = ButlerURI(uri_str, forceDirectory=True)
174 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}")
175 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI")
177 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c")
178 uriStrings = (
179 ("file://localhost/file.ext", "file.ext"),
180 (exampleLocalFile, os.path.join("a", "b", "c")),
181 ("s3://bucket/path/file.ext", "path/file.ext"),
182 ("https://host.com/a/b/c.d", "a/b/c.d"),
183 )
185 for uri_str, result in uriStrings:
186 uri = ButlerURI(uri_str)
187 self.assertEqual(uri.relativeToPathRoot, result)
189 def testUriJoin(self):
190 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False)
191 uri2 = uri.join("e/f/g.txt")
192 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
194 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False)
195 uri2 = uri.join("e/f/g.txt")
196 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
198 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True)
199 uri2 = uri.join("e/f/g.txt")
200 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
202 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True)
203 uri2 = uri.join("newpath/newfile.txt")
204 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
206 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt")
207 uri2 = uri.join("newpath/newfile.txt")
208 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
210 def testButlerUriSerialization(self):
211 """Test that we can pickle and yaml"""
212 uri = ButlerURI("a/b/c/d")
213 uri2 = pickle.loads(pickle.dumps(uri))
214 self.assertEqual(uri, uri2)
215 self.assertFalse(uri2.dirLike)
217 uri = ButlerURI("a/b/c/d", forceDirectory=True)
218 uri2 = pickle.loads(pickle.dumps(uri))
219 self.assertEqual(uri, uri2)
220 self.assertTrue(uri2.dirLike)
222 def testUriExtensions(self):
223 """Test extension extraction."""
225 files = (
226 ("file.fits.gz", ".fits.gz"),
227 ("file.fits", ".fits"),
228 ("file.fits.xz", ".fits.xz"),
229 ("file.fits.tar", ".tar"),
230 ("file", ""),
231 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"),
232 ("flat_i_sim_1.4_blah.txt", ".txt"),
233 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"),
234 ("flat_i_sim_1.4_blah.fits.txt", ".txt"),
235 ("s3://bucket/c/a.b/", ""),
236 ("s3://bucket/c/a.b", ".b"),
237 ("file://localhost/c/a.b.gz", ".b.gz"),
238 )
240 for file, expected in files:
241 test_string = file
242 if ":" not in test_string:
243 test_string = f"a/b/{test_string}"
244 uri = ButlerURI(test_string)
245 self.assertEqual(uri.getExtension(), expected)
247 def testFileLocation(self):
248 root = os.path.abspath(os.path.curdir)
249 factory = LocationFactory(root)
250 print(f"Factory created: {factory}")
252 pathInStore = "relative/path/file.ext"
253 loc1 = factory.fromPath(pathInStore)
255 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
256 self.assertEqual(loc1.pathInStore.path, pathInStore)
257 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
258 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
259 loc1.updateExtension("fits")
260 self.assertTrue(loc1.uri.geturl().endswith("file.fits"), f"Checking 'fits' extension in {loc1.uri}")
261 loc1.updateExtension("fits.gz")
262 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
263 self.assertTrue(
264 loc1.uri.geturl().endswith("file.fits.gz"), f"Checking 'fits.gz' extension in {loc1.uri}"
265 )
266 self.assertEqual(loc1.getExtension(), ".fits.gz")
267 loc1.updateExtension(".jpeg")
268 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"), f"Checking 'jpeg' extension in {loc1.uri}")
269 loc1.updateExtension(None)
270 self.assertTrue(
271 loc1.uri.geturl().endswith("file.jpeg"), f"Checking unchanged extension in {loc1.uri}"
272 )
273 loc1.updateExtension("")
274 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
275 self.assertEqual(loc1.getExtension(), "")
277 loc2 = factory.fromPath(pathInStore)
278 loc3 = factory.fromPath(pathInStore)
279 self.assertEqual(loc2, loc3)
281 def testAbsoluteLocations(self):
282 """Using a pathInStore that refers to absolute URI."""
283 loc = Location(None, "file:///something.txt")
284 self.assertEqual(loc.pathInStore.path, "/something.txt")
285 self.assertEqual(str(loc.uri), "file:///something.txt")
287 with self.assertRaises(ValueError):
288 Location(None, "relative.txt")
290 def testRelativeRoot(self):
291 root = os.path.abspath(os.path.curdir)
292 factory = LocationFactory(os.path.curdir)
294 pathInStore = "relative/path/file.ext"
295 loc1 = factory.fromPath(pathInStore)
297 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
298 self.assertEqual(loc1.pathInStore.path, pathInStore)
299 self.assertEqual(loc1.uri.scheme, "file")
301 with self.assertRaises(ValueError):
302 factory.fromPath("../something")
304 def testQuotedRoot(self):
305 """Test we can handle quoted characters."""
306 root = "/a/b/c+1/d"
307 factory = LocationFactory(root)
309 pathInStore = "relative/path/file.ext.gz"
311 for pathInStore in (
312 "relative/path/file.ext.gz",
313 "relative/path+2/file.ext.gz",
314 "relative/path+3/file&.ext.gz",
315 ):
316 loc1 = factory.fromPath(pathInStore)
318 self.assertEqual(loc1.pathInStore.path, pathInStore)
319 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
320 self.assertIn("%", str(loc1.uri))
321 self.assertEqual(loc1.getExtension(), ".ext.gz")
323 def testHttpLocation(self):
324 root = "https://www.lsst.org/butler/datastore"
325 factory = LocationFactory(root)
326 print(f"Factory created: {factory}")
328 pathInStore = "relative/path/file.ext"
329 loc1 = factory.fromPath(pathInStore)
331 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
332 self.assertEqual(loc1.pathInStore.path, pathInStore)
333 self.assertEqual(loc1.uri.scheme, "https")
334 self.assertEqual(loc1.uri.basename(), "file.ext")
335 loc1.updateExtension("fits")
336 self.assertTrue(loc1.uri.basename(), "file.fits")
338 def testPosix2OS(self):
339 """Test round tripping of the posix to os.path conversion helpers."""
340 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
341 for p in testPaths:
342 with self.subTest(path=p):
343 self.assertEqual(os2posix(posix2os(p)), p)
345 def testSplit(self):
346 """Tests split functionality."""
347 testRoot = "/tmp/"
349 testPaths = (
350 "/absolute/file.ext",
351 "/absolute/",
352 "file:///absolute/file.ext",
353 "file:///absolute/",
354 "s3://bucket/root/file.ext",
355 "s3://bucket/root/",
356 "https://www.lsst.org/root/file.ext",
357 "https://www.lsst.org/root/",
358 "relative/file.ext",
359 "relative/",
360 )
362 osRelExpected = os.path.join(testRoot, "relative")
363 expected = (
364 ("file:///absolute/", "file.ext"),
365 ("file:///absolute/", ""),
366 ("file:///absolute/", "file.ext"),
367 ("file:///absolute/", ""),
368 ("s3://bucket/root/", "file.ext"),
369 ("s3://bucket/root/", ""),
370 ("https://www.lsst.org/root/", "file.ext"),
371 ("https://www.lsst.org/root/", ""),
372 (f"file://{osRelExpected}/", "file.ext"),
373 (f"file://{osRelExpected}/", ""),
374 )
376 for p, e in zip(testPaths, expected):
377 with self.subTest(path=p):
378 uri = ButlerURI(p, testRoot)
379 head, tail = uri.split()
380 self.assertEqual((head.geturl(), tail), e)
382 # explicit file scheme should force posixpath, check os.path is ignored
383 posixRelFilePath = posixpath.join(testRoot, "relative")
384 uri = ButlerURI("file:relative/file.ext", testRoot)
385 head, tail = uri.split()
386 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
388 # check head can be empty and we do not get an absolute path back
389 uri = ButlerURI("file.ext", forceAbsolute=False)
390 head, tail = uri.split()
391 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
393 # ensure empty path splits to a directory URL
394 uri = ButlerURI("", forceAbsolute=False)
395 head, tail = uri.split()
396 self.assertEqual((head.geturl(), tail), ("./", ""))
398 uri = ButlerURI(".", forceAbsolute=False)
399 head, tail = uri.split()
400 self.assertEqual((head.geturl(), tail), ("./", ""))
403if __name__ == "__main__": 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true
404 unittest.main()