Coverage for tests/test_location.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import copy
23import unittest
24import os.path
25import posixpath
26import pickle
27import pathlib
29from lsst.daf.butler import LocationFactory, Location, ButlerURI
30from lsst.daf.butler.core._butlerUri.utils import os2posix, posix2os
33class LocationTestCase(unittest.TestCase):
34 """Tests for Location within datastore
35 """
37 def testButlerUri(self):
38 """Tests whether ButlerURI instantiates correctly given different
39 arguments.
40 """
41 # Root to use for relative paths
42 testRoot = "/tmp/"
44 # uriStrings is a list of tuples containing test string, forceAbsolute,
45 # forceDirectory as arguments to ButlerURI and scheme, netloc and path
46 # as expected attributes. Test asserts constructed equals to expected.
47 # 1) no determinable schemes (ensures schema and netloc are not set)
48 osRelFilePath = os.path.join(testRoot, "relative/file.ext")
49 uriStrings = [
50 ("relative/file.ext", True, False, "", "", osRelFilePath),
51 ("relative/file.ext", False, False, "", "", "relative/file.ext"),
52 ("test/../relative/file.ext", True, False, "", "", osRelFilePath),
53 ("test/../relative/file.ext", False, False, "", "", "relative/file.ext"),
54 ("relative/dir", False, True, "", "", "relative/dir/")
55 ]
56 # 2) implicit file scheme, tests absolute file and directory paths
57 uriStrings.extend((
58 ("/rootDir/absolute/file.ext", True, False, "file", "", '/rootDir/absolute/file.ext'),
59 ("~/relative/file.ext", True, False, "file", "", os.path.expanduser("~/relative/file.ext")),
60 ("~/relative/file.ext", False, False, "file", "", os.path.expanduser("~/relative/file.ext")),
61 ("/rootDir/absolute/", True, False, "file", "", "/rootDir/absolute/"),
62 ("/rootDir/absolute", True, True, "file", "", "/rootDir/absolute/"),
63 ("~/rootDir/absolute", True, True, "file", "", os.path.expanduser("~/rootDir/absolute/"))
64 ))
65 # 3) explicit file scheme, absolute and relative file and directory URI
66 posixRelFilePath = posixpath.join(testRoot, "relative/file.ext")
67 uriStrings.extend((
68 ("file:///rootDir/absolute/file.ext", True, False, "file", "", "/rootDir/absolute/file.ext"),
69 ("file:relative/file.ext", True, False, "file", "", posixRelFilePath),
70 ("file:///absolute/directory/", True, False, "file", "", "/absolute/directory/"),
71 ("file:///absolute/directory", True, True, "file", "", "/absolute/directory/")
72 ))
73 # 4) S3 scheme (ensured Keys as dirs and fully specified URIs work)
74 uriStrings.extend((
75 ("s3://bucketname/rootDir/", True, False, "s3", "bucketname", "/rootDir/"),
76 ("s3://bucketname/rootDir", True, True, "s3", "bucketname", "/rootDir/"),
77 ("s3://bucketname/rootDir/relative/file.ext", True, False, "s3",
78 "bucketname", "/rootDir/relative/file.ext")
79 ))
80 # 5) HTTPS scheme
81 uriStrings.extend((
82 ("https://www.lsst.org/rootDir/", True, False, "https", "www.lsst.org", "/rootDir/"),
83 ("https://www.lsst.org/rootDir", True, True, "https", "www.lsst.org", "/rootDir/"),
84 ("https://www.lsst.org/rootDir/relative/file.ext", True, False, "https",
85 "www.lsst.org", "/rootDir/relative/file.ext")
86 ))
88 for uriInfo in uriStrings:
89 uri = ButlerURI(uriInfo[0], root=testRoot, forceAbsolute=uriInfo[1],
90 forceDirectory=uriInfo[2])
91 with self.subTest(uri=uriInfo[0]):
92 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
93 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
94 self.assertEqual(uri.path, uriInfo[5], "test path")
96 # test root becomes abspath(".") when not specified, note specific
97 # file:// scheme case
98 uriStrings = (
99 ("file://relative/file.ext", True, False, "file", "relative", "/file.ext"),
100 ("file:relative/file.ext", False, False, "file", "", os.path.abspath("relative/file.ext")),
101 ("file:relative/dir/", True, True, "file", "", os.path.abspath("relative/dir")+"/"),
102 ("relative/file.ext", True, False, "", "", os.path.abspath("relative/file.ext"))
103 )
105 for uriInfo in uriStrings:
106 uri = ButlerURI(uriInfo[0], forceAbsolute=uriInfo[1], forceDirectory=uriInfo[2])
107 with self.subTest(uri=uriInfo[0]):
108 self.assertEqual(uri.scheme, uriInfo[3], "test scheme")
109 self.assertEqual(uri.netloc, uriInfo[4], "test netloc")
110 self.assertEqual(uri.path, uriInfo[5], "test path")
112 # File replacement
113 uriStrings = (
114 ("relative/file.ext", "newfile.fits", "relative/newfile.fits"),
115 ("relative/", "newfile.fits", "relative/newfile.fits"),
116 ("https://www.lsst.org/butler/", "butler.yaml", "/butler/butler.yaml"),
117 ("s3://amazon/datastore/", "butler.yaml", "/datastore/butler.yaml"),
118 ("s3://amazon/datastore/mybutler.yaml", "butler.yaml", "/datastore/butler.yaml")
119 )
121 for uriInfo in uriStrings:
122 uri = ButlerURI(uriInfo[0], forceAbsolute=False)
123 uri.updateFile(uriInfo[1])
124 with self.subTest(uri=uriInfo[0]):
125 self.assertEqual(uri.path, uriInfo[2])
127 # Check that schemeless can become file scheme
128 schemeless = ButlerURI("relative/path.ext")
129 filescheme = ButlerURI("/absolute/path.ext")
130 self.assertFalse(schemeless.scheme)
131 self.assertEqual(filescheme.scheme, "file")
132 self.assertNotEqual(type(schemeless), type(filescheme))
134 # Copy constructor
135 uri = ButlerURI("s3://amazon/datastore", forceDirectory=True)
136 uri2 = ButlerURI(uri)
137 self.assertEqual(uri, uri2)
138 uri = ButlerURI("file://amazon/datastore/file.txt")
139 uri2 = ButlerURI(uri)
140 self.assertEqual(uri, uri2)
142 # Copy constructor using subclass
143 uri3 = type(uri)(uri)
144 self.assertEqual(type(uri), type(uri3))
146 # Explicit copy
147 uri4 = copy.copy(uri3)
148 self.assertEqual(uri4, uri3)
149 uri4 = copy.deepcopy(uri3)
150 self.assertEqual(uri4, uri3)
152 def testUriRoot(self):
153 osPathRoot = pathlib.Path(__file__).absolute().root
154 rootUris = (osPathRoot, "s3://bucket", "file://localhost/", "https://a.b.com")
155 for uri_str in rootUris:
156 uri = ButlerURI(uri_str, forceDirectory=True)
157 self.assertEqual(uri.relativeToPathRoot, "./", f"Testing uri: {uri}")
158 self.assertTrue(uri.is_root, f"Testing URI {uri} is a root URI")
160 exampleLocalFile = os.path.join(osPathRoot, "a", "b", "c")
161 uriStrings = (
162 ("file://localhost/file.ext", "file.ext"),
163 (exampleLocalFile, os.path.join("a", "b", "c")),
164 ("s3://bucket/path/file.ext", "path/file.ext"),
165 ("https://host.com/a/b/c.d", "a/b/c.d"),
166 )
168 for uri_str, result in uriStrings:
169 uri = ButlerURI(uri_str)
170 self.assertEqual(uri.relativeToPathRoot, result)
172 def testUriJoin(self):
173 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=False)
174 uri2 = uri.join("e/f/g.txt")
175 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
177 uri = ButlerURI("a/b/c/d/old.txt", forceAbsolute=False)
178 uri2 = uri.join("e/f/g.txt")
179 self.assertEqual(str(uri2), "a/b/c/d/e/f/g.txt", f"Checking joined URI {uri} -> {uri2}")
181 uri = ButlerURI("a/b/c/d", forceDirectory=True, forceAbsolute=True)
182 uri2 = uri.join("e/f/g.txt")
183 self.assertTrue(str(uri2).endswith("a/b/c/d/e/f/g.txt"), f"Checking joined URI {uri} -> {uri2}")
185 uri = ButlerURI("s3://bucket/a/b/c/d", forceDirectory=True)
186 uri2 = uri.join("newpath/newfile.txt")
187 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
189 uri = ButlerURI("s3://bucket/a/b/c/d/old.txt")
190 uri2 = uri.join("newpath/newfile.txt")
191 self.assertEqual(str(uri2), "s3://bucket/a/b/c/d/newpath/newfile.txt")
193 def testButlerUriSerialization(self):
194 """Test that we can pickle and yaml"""
195 uri = ButlerURI("a/b/c/d")
196 uri2 = pickle.loads(pickle.dumps(uri))
197 self.assertEqual(uri, uri2)
198 self.assertFalse(uri2.dirLike)
200 uri = ButlerURI("a/b/c/d", forceDirectory=True)
201 uri2 = pickle.loads(pickle.dumps(uri))
202 self.assertEqual(uri, uri2)
203 self.assertTrue(uri2.dirLike)
205 def testUriExtensions(self):
206 """Test extension extraction."""
208 files = (("file.fits.gz", ".fits.gz"),
209 ("file.fits", ".fits"),
210 ("file.fits.xz", ".fits.xz"),
211 ("file.fits.tar", ".tar"),
212 ("file", ""),
213 ("flat_i_sim_1.4_blah.fits.gz", ".fits.gz"),
214 ("flat_i_sim_1.4_blah.txt", ".txt"),
215 ("flat_i_sim_1.4_blah.fits.fz", ".fits.fz"),
216 ("flat_i_sim_1.4_blah.fits.txt", ".txt"),
217 )
219 for file, expected in files:
220 uri = ButlerURI(f"a/b/{file}")
221 self.assertEqual(uri.getExtension(), expected)
223 def testFileLocation(self):
224 root = os.path.abspath(os.path.curdir)
225 factory = LocationFactory(root)
226 print(f"Factory created: {factory}")
228 pathInStore = "relative/path/file.ext"
229 loc1 = factory.fromPath(pathInStore)
231 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
232 self.assertEqual(loc1.pathInStore.path, pathInStore)
233 self.assertTrue(loc1.uri.geturl().startswith("file:///"))
234 self.assertTrue(loc1.uri.geturl().endswith("file.ext"))
235 loc1.updateExtension("fits")
236 self.assertTrue(loc1.uri.geturl().endswith("file.fits"),
237 f"Checking 'fits' extension in {loc1.uri}")
238 loc1.updateExtension("fits.gz")
239 self.assertEqual(loc1.uri.basename(), "file.fits.gz")
240 self.assertTrue(loc1.uri.geturl().endswith("file.fits.gz"),
241 f"Checking 'fits.gz' extension in {loc1.uri}")
242 self.assertEqual(loc1.getExtension(), ".fits.gz")
243 loc1.updateExtension(".jpeg")
244 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"),
245 f"Checking 'jpeg' extension in {loc1.uri}")
246 loc1.updateExtension(None)
247 self.assertTrue(loc1.uri.geturl().endswith("file.jpeg"),
248 f"Checking unchanged extension in {loc1.uri}")
249 loc1.updateExtension("")
250 self.assertTrue(loc1.uri.geturl().endswith("file"), f"Checking no extension in {loc1.uri}")
251 self.assertEqual(loc1.getExtension(), "")
253 loc2 = factory.fromPath(pathInStore)
254 loc3 = factory.fromPath(pathInStore)
255 self.assertEqual(loc2, loc3)
257 def testAbsoluteLocations(self):
258 """Using a pathInStore that refers to absolute URI."""
259 loc = Location(None, "file:///something.txt")
260 self.assertEqual(loc.pathInStore.path, "/something.txt")
261 self.assertEqual(str(loc.uri), "file:///something.txt")
263 with self.assertRaises(ValueError):
264 Location(None, "relative.txt")
266 def testRelativeRoot(self):
267 root = os.path.abspath(os.path.curdir)
268 factory = LocationFactory(os.path.curdir)
270 pathInStore = "relative/path/file.ext"
271 loc1 = factory.fromPath(pathInStore)
273 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
274 self.assertEqual(loc1.pathInStore.path, pathInStore)
275 self.assertEqual(loc1.uri.scheme, "file")
277 with self.assertRaises(ValueError):
278 factory.fromPath("../something")
280 def testQuotedRoot(self):
281 """Test we can handle quoted characters."""
282 root = "/a/b/c+1/d"
283 factory = LocationFactory(root)
285 pathInStore = "relative/path/file.ext.gz"
287 for pathInStore in ("relative/path/file.ext.gz",
288 "relative/path+2/file.ext.gz",
289 "relative/path+3/file&.ext.gz"):
290 loc1 = factory.fromPath(pathInStore)
292 self.assertEqual(loc1.pathInStore.path, pathInStore)
293 self.assertEqual(loc1.path, os.path.join(root, pathInStore))
294 self.assertIn("%", str(loc1.uri))
295 self.assertEqual(loc1.getExtension(), ".ext.gz")
297 def testHttpLocation(self):
298 root = "https://www.lsst.org/butler/datastore"
299 factory = LocationFactory(root)
300 print(f"Factory created: {factory}")
302 pathInStore = "relative/path/file.ext"
303 loc1 = factory.fromPath(pathInStore)
305 self.assertEqual(loc1.path, posixpath.join("/butler/datastore", pathInStore))
306 self.assertEqual(loc1.pathInStore.path, pathInStore)
307 self.assertEqual(loc1.uri.scheme, "https")
308 self.assertEqual(loc1.uri.basename(), "file.ext")
309 loc1.updateExtension("fits")
310 self.assertTrue(loc1.uri.basename(), "file.fits")
312 def testPosix2OS(self):
313 """Test round tripping of the posix to os.path conversion helpers."""
314 testPaths = ("/a/b/c.e", "a/b", "a/b/", "/a/b", "/a/b/", "a/b/c.e")
315 for p in testPaths:
316 with self.subTest(path=p):
317 self.assertEqual(os2posix(posix2os(p)), p)
319 def testSplit(self):
320 """Tests split functionality."""
321 testRoot = "/tmp/"
323 testPaths = ("/absolute/file.ext", "/absolute/",
324 "file:///absolute/file.ext", "file:///absolute/",
325 "s3://bucket/root/file.ext", "s3://bucket/root/",
326 "https://www.lsst.org/root/file.ext", "https://www.lsst.org/root/",
327 "relative/file.ext", "relative/")
329 osRelExpected = os.path.join(testRoot, "relative")
330 expected = (("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
331 ("file:///absolute/", "file.ext"), ("file:///absolute/", ""),
332 ("s3://bucket/root/", "file.ext"), ("s3://bucket/root/", ""),
333 ("https://www.lsst.org/root/", "file.ext"), ("https://www.lsst.org/root/", ""),
334 (f"file://{osRelExpected}/", "file.ext"), (f"file://{osRelExpected}/", ""))
336 for p, e in zip(testPaths, expected):
337 with self.subTest(path=p):
338 uri = ButlerURI(p, testRoot)
339 head, tail = uri.split()
340 self.assertEqual((head.geturl(), tail), e)
342 # explicit file scheme should force posixpath, check os.path is ignored
343 posixRelFilePath = posixpath.join(testRoot, "relative")
344 uri = ButlerURI("file:relative/file.ext", testRoot)
345 head, tail = uri.split()
346 self.assertEqual((head.geturl(), tail), (f"file://{posixRelFilePath}/", "file.ext"))
348 # check head can be empty and we do not get an absolute path back
349 uri = ButlerURI("file.ext", forceAbsolute=False)
350 head, tail = uri.split()
351 self.assertEqual((head.geturl(), tail), ("./", "file.ext"))
353 # ensure empty path splits to a directory URL
354 uri = ButlerURI("", forceAbsolute=False)
355 head, tail = uri.split()
356 self.assertEqual((head.geturl(), tail), ("./", ""))
358 uri = ButlerURI(".", forceAbsolute=False)
359 head, tail = uri.split()
360 self.assertEqual((head.geturl(), tail), ("./", ""))
363if __name__ == "__main__": 363 ↛ 364line 363 didn't jump to line 364, because the condition on line 363 was never true
364 unittest.main()