Coverage for tests/test_uri.py : 15%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import os
23import shutil
24import tempfile
25import unittest
26import urllib.parse
28try:
29 import boto3
30 import botocore
31 from moto import mock_s3
32except ImportError:
33 boto3 = None
35 def mock_s3(cls):
36 """A no-op decorator in case moto mock_s3 can not be imported.
37 """
38 return cls
40from lsst.daf.butler import ButlerURI
41from lsst.daf.butler.core.s3utils import (setAwsEnvCredentials,
42 unsetAwsEnvCredentials)
44TESTDIR = os.path.abspath(os.path.dirname(__file__))
47class FileURITestCase(unittest.TestCase):
48 """Concrete tests for local files"""
50 def setUp(self):
51 # Use a local tempdir because on macOS the temp dirs use symlinks
52 # so relsymlink gets quite confused.
53 self.tmpdir = tempfile.mkdtemp(dir=TESTDIR)
55 def tearDown(self):
56 shutil.rmtree(self.tmpdir, ignore_errors=True)
58 def testFile(self):
59 file = os.path.join(self.tmpdir, "test.txt")
60 uri = ButlerURI(file)
61 self.assertFalse(uri.exists(), f"{uri} should not exist")
62 self.assertEqual(uri.ospath, file)
64 content = "abcdefghijklmnopqrstuv\n"
65 uri.write(content.encode())
66 self.assertTrue(os.path.exists(file), "File should exist locally")
67 self.assertTrue(uri.exists(), f"{uri} should now exist")
68 self.assertEqual(uri.read().decode(), content)
70 def testRelative(self):
71 """Check that we can get subpaths back from two URIs"""
72 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True)
73 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True)
75 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
77 not_child = ButlerURI("/a/b/dir1/file.txt")
78 self.assertFalse(not_child.relative_to(parent))
80 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
81 self.assertFalse(child.relative_to(not_directory))
83 # Relative URIs
84 parent = ButlerURI("a/b/", forceAbsolute=False)
85 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False)
86 self.assertFalse(child.scheme)
87 self.assertEqual(child.relative_to(parent), "c/d.txt")
89 # File URI and schemeless URI
90 parent = ButlerURI("file:/a/b/c/")
91 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
93 # If the child is relative and the parent is absolute we assume
94 # that the child is a child of the parent unless it uses ".."
95 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
97 child = ButlerURI("../e/f/g.txt", forceAbsolute=False)
98 self.assertFalse(child.relative_to(parent))
100 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False)
101 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
103 def testMkdir(self):
104 tmpdir = ButlerURI(self.tmpdir)
105 newdir = tmpdir.join("newdir/seconddir")
106 newdir.mkdir()
107 self.assertTrue(newdir.exists())
108 newfile = newdir.join("temp.txt")
109 newfile.write("Data".encode())
110 self.assertTrue(newfile.exists())
112 def testTransfer(self):
113 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
114 content = "Content is some content\nwith something to say\n\n"
115 src.write(content.encode())
117 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
118 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
119 dest.transfer_from(src, transfer=mode)
120 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
122 with open(dest.ospath, "r") as fh:
123 new_content = fh.read()
124 self.assertEqual(new_content, content)
126 if mode in ("symlink", "relsymlink"):
127 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
129 with self.assertRaises(FileExistsError):
130 dest.transfer_from(src, transfer=mode)
132 dest.transfer_from(src, transfer=mode, overwrite=True)
134 os.remove(dest.ospath)
136 b = src.read()
137 self.assertEqual(b.decode(), new_content)
139 nbytes = 10
140 subset = src.read(size=nbytes)
141 self.assertEqual(len(subset), nbytes)
142 self.assertEqual(subset.decode(), content[:nbytes])
144 with self.assertRaises(ValueError):
145 src.transfer_from(src, transfer="unknown")
147 def testResource(self):
148 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml")
149 self.assertTrue(u.exists(), f"Check {u} exists")
151 content = u.read().decode()
152 self.assertTrue(content.startswith("datastore:"))
154 truncated = u.read(size=9).decode()
155 self.assertEqual(truncated, "datastore")
157 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True)
158 self.assertTrue(u.exists(), f"Check directory {d} exists")
160 j = d.join("datastore.yaml")
161 self.assertEqual(u, j)
162 self.assertFalse(j.dirLike)
163 self.assertFalse(d.join("not-there.yaml").exists())
165 def testEscapes(self):
166 """Special characters in file paths"""
167 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
168 self.assertFalse(src.scheme)
169 src.write(b"Some content")
170 self.assertTrue(src.exists())
172 # Use the internal API to force to a file
173 file = src._force_to_file()
174 self.assertTrue(file.exists())
175 self.assertIn("???", file.ospath)
176 self.assertNotIn("???", file.path)
178 file.updateFile("tests??.txt")
179 self.assertNotIn("??.txt", file.path)
180 file.write(b"Other content")
181 self.assertEqual(file.read(), b"Other content")
183 src.updateFile("tests??.txt")
184 self.assertIn("??.txt", src.path)
185 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
187 # File URI and schemeless URI
188 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/"))
189 child = ButlerURI("e/f/g.txt", forceAbsolute=False)
190 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
192 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False)
193 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
195 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
196 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
198 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
200 # Schemeless so should not quote
201 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
202 self.assertIn("???", dir.ospath)
203 self.assertIn("???", dir.path)
204 self.assertFalse(dir.scheme)
206 # dir.join() morphs into a file scheme
207 new = dir.join("test_j.txt")
208 self.assertIn("???", new.ospath, f"Checking {new}")
209 new.write(b"Content")
211 new2name = "###/test??.txt"
212 new2 = dir.join(new2name)
213 self.assertIn("???", new2.ospath)
214 new2.write(b"Content")
215 self.assertTrue(new2.ospath.endswith(new2name))
216 self.assertEqual(new.read(), new2.read())
218 fdir = dir._force_to_file()
219 self.assertNotIn("???", fdir.path)
220 self.assertIn("???", fdir.ospath)
221 self.assertEqual(fdir.scheme, "file")
222 fnew = dir.join("test_jf.txt")
223 fnew.write(b"Content")
225 fnew2 = fdir.join(new2name)
226 fnew2.write(b"Content")
227 self.assertTrue(fnew2.ospath.endswith(new2name))
228 self.assertNotIn("###", fnew2.path)
230 self.assertEqual(fnew.read(), fnew2.read())
232 # Test that children relative to schemeless and file schemes
233 # still return the same unquoted name
234 self.assertEqual(fnew2.relative_to(fdir), new2name)
235 self.assertEqual(fnew2.relative_to(dir), new2name)
236 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}")
237 self.assertEqual(new2.relative_to(dir), new2name)
239 # Check for double quoting
240 plus_path = "/a/b/c+d/"
241 with self.assertLogs(level="WARNING"):
242 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True)
243 self.assertEqual(uri.ospath, plus_path)
246@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
247@mock_s3
248class S3URITestCase(unittest.TestCase):
249 """Tests involving S3"""
251 bucketName = "any_bucket"
252 """Bucket name to use in tests"""
254 def setUp(self):
255 # Local test directory
256 self.tmpdir = tempfile.mkdtemp()
258 # set up some fake credentials if they do not exist
259 self.usingDummyCredentials = setAwsEnvCredentials()
261 # MOTO needs to know that we expect Bucket bucketname to exist
262 s3 = boto3.resource("s3")
263 s3.create_bucket(Bucket=self.bucketName)
265 def tearDown(self):
266 s3 = boto3.resource("s3")
267 bucket = s3.Bucket(self.bucketName)
268 try:
269 bucket.objects.all().delete()
270 except botocore.exceptions.ClientError as e:
271 if e.response["Error"]["Code"] == "404":
272 # the key was not reachable - pass
273 pass
274 else:
275 raise
277 bucket = s3.Bucket(self.bucketName)
278 bucket.delete()
280 # unset any potentially set dummy credentials
281 if self.usingDummyCredentials:
282 unsetAwsEnvCredentials()
284 shutil.rmtree(self.tmpdir, ignore_errors=True)
286 def makeS3Uri(self, path):
287 return f"s3://{self.bucketName}/{path}"
289 def testTransfer(self):
290 src = ButlerURI(os.path.join(self.tmpdir, "test.txt"))
291 content = "Content is some content\nwith something to say\n\n"
292 src.write(content.encode())
294 dest = ButlerURI(self.makeS3Uri("test.txt"))
295 self.assertFalse(dest.exists())
296 dest.transfer_from(src, transfer="copy")
297 self.assertTrue(dest.exists())
299 dest2 = ButlerURI(self.makeS3Uri("copied.txt"))
300 dest2.transfer_from(dest, transfer="copy")
301 self.assertTrue(dest2.exists())
303 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt"))
304 local.transfer_from(dest2, transfer="copy")
305 with open(local.ospath, "r") as fd:
306 new_content = fd.read()
307 self.assertEqual(new_content, content)
309 with self.assertRaises(ValueError):
310 dest2.transfer_from(local, transfer="symlink")
312 b = dest.read()
313 self.assertEqual(b.decode(), new_content)
315 nbytes = 10
316 subset = dest.read(size=nbytes)
317 self.assertEqual(len(subset), nbytes) # Extra byte comes back
318 self.assertEqual(subset.decode(), content[:nbytes])
320 with self.assertRaises(FileExistsError):
321 dest.transfer_from(src, transfer="copy")
323 dest.transfer_from(src, transfer="copy", overwrite=True)
325 def testWrite(self):
326 s3write = ButlerURI(self.makeS3Uri("created.txt"))
327 content = "abcdefghijklmnopqrstuv\n"
328 s3write.write(content.encode())
329 self.assertEqual(s3write.read().decode(), content)
331 def testRelative(self):
332 """Check that we can get subpaths back from two URIs"""
333 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
334 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt"))
336 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
338 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt"))
339 self.assertFalse(not_child.relative_to(parent))
341 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt"))
342 self.assertFalse(child.relative_to(not_s3))
344 def testQuoting(self):
345 """Check that quoting works."""
346 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True)
347 subpath = "rootdir/dir1+/file?.txt"
348 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath)))
350 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
351 self.assertEqual(child.basename(), "file?.txt")
352 self.assertEqual(child.relativeToPathRoot, subpath)
353 self.assertIn("%", child.path)
354 self.assertEqual(child.unquoted_path, "/" + subpath)
357if __name__ == "__main__": 357 ↛ 358line 357 didn't jump to line 358, because the condition on line 357 was never true
358 unittest.main()