Coverage for tests/test_uri.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import glob
13import os
14import pathlib
15import shutil
16import unittest
17import urllib.parse
18import uuid
20import responses
22try:
23 import boto3
24 import botocore
25 from moto import mock_s3
26except ImportError:
27 boto3 = None
29 def mock_s3(cls):
30 """A no-op decorator in case moto mock_s3 can not be imported."""
31 return cls
34import lsst.resources
35from lsst.resources import ResourcePath
36from lsst.resources.s3utils import setAwsEnvCredentials, unsetAwsEnvCredentials
37from lsst.resources.utils import makeTestTempDir, removeTestTempDir
39TESTDIR = os.path.abspath(os.path.dirname(__file__))
42def _check_open(test_case, uri, *, mode_suffixes=("", "t", "b"), **kwargs) -> None:
43 """Test an implementation of ButlerURI.open.
45 Parameters
46 ----------
47 test_case : `unittest.TestCase`
48 Test case to use for assertions.
49 uri : `ButlerURI`
50 URI to use for tests. Must point to a writeable location that is not
51 yet occupied by a file. On return, the location may point to a file
52 only if the test fails.
53 mode_suffixes : `Iterable` of `str`
54 Suffixes to pass as part of the ``mode`` argument to `ButlerURI.open`,
55 indicating whether to open as binary or as text; the only permitted
56 elements are ``""``, ``"t"``, and ``""b""`.
57 **kwargs
58 Additional keyword arguments to forward to all calls to `open`.
59 """
60 text_content = "wxyz🙂"
61 bytes_content = uuid.uuid4().bytes
62 content_by_mode_suffix = {
63 "": text_content,
64 "t": text_content,
65 "b": bytes_content,
66 }
67 empty_content_by_mode_suffix = {
68 "": "",
69 "t": "",
70 "b": b"",
71 }
72 for mode_suffix in mode_suffixes:
73 content = content_by_mode_suffix[mode_suffix]
74 # Create file with mode='x', which prohibits overwriting.
75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
76 write_buffer.write(content)
77 test_case.assertTrue(uri.exists())
78 # Check that opening with 'x' now raises, and does not modify content.
79 with test_case.assertRaises(FileExistsError):
80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
81 write_buffer.write("bad")
82 # Read the file we created and check the contents.
83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
84 test_case.assertEqual(read_buffer.read(), content)
85 # Write two copies of the content, overwriting the single copy there.
86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
87 write_buffer.write(content + content)
88 # Read again, this time use mode='r+', which reads what is there and
89 # then lets us write more; we'll use that to reset the file to one
90 # copy of the content.
91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
92 test_case.assertEqual(rw_buffer.read(), content + content)
93 rw_buffer.seek(0)
94 rw_buffer.truncate()
95 rw_buffer.write(content)
96 rw_buffer.seek(0)
97 test_case.assertEqual(rw_buffer.read(), content)
98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
99 test_case.assertEqual(read_buffer.read(), content)
100 # Append some more content to the file; should now have two copies.
101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
102 append_buffer.write(content)
103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
104 test_case.assertEqual(read_buffer.read(), content + content)
105 # Final mode to check is w+, which does read/write but truncates first.
106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
108 rw_buffer.write(content)
109 rw_buffer.seek(0)
110 test_case.assertEqual(rw_buffer.read(), content)
111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
112 test_case.assertEqual(read_buffer.read(), content)
113 # Remove file to make room for the next loop of tests with this URI.
114 uri.remove()
117class FileURITestCase(unittest.TestCase):
118 """Concrete tests for local files."""
120 def setUp(self):
121 # Use a local tempdir because on macOS the temp dirs use symlinks
122 # so relsymlink gets quite confused.
123 self.tmpdir = makeTestTempDir(TESTDIR)
125 def tearDown(self):
126 removeTestTempDir(self.tmpdir)
128 def testFile(self):
129 file = os.path.join(self.tmpdir, "test.txt")
130 uri = ResourcePath(file)
131 self.assertFalse(uri.exists(), f"{uri} should not exist")
132 self.assertEqual(uri.ospath, file)
134 path = pathlib.Path(file)
135 uri = ResourcePath(path)
136 self.assertEqual(uri.ospath, file)
138 content = "abcdefghijklmnopqrstuv\n"
139 uri.write(content.encode())
140 self.assertTrue(os.path.exists(file), "File should exist locally")
141 self.assertTrue(uri.exists(), f"{uri} should now exist")
142 self.assertEqual(uri.read().decode(), content)
143 self.assertEqual(uri.size(), len(content.encode()))
145 with self.assertRaises(FileNotFoundError):
146 ResourcePath("file/not/there.txt").size()
148 # Check that creating a URI from a URI returns the same thing
149 uri2 = ResourcePath(uri)
150 self.assertEqual(uri, uri2)
151 self.assertEqual(id(uri), id(uri2))
153 with self.assertRaises(ValueError):
154 # Scheme-less URIs are not allowed to support non-file roots
155 # at the present time. This may change in the future to become
156 # equivalent to ResourcePath.join()
157 ResourcePath("a/b.txt", root=ResourcePath("s3://bucket/a/b/"))
159 def testExtension(self):
160 file = ResourcePath(os.path.join(self.tmpdir, "test.txt"))
161 self.assertEqual(file.updatedExtension(None), file)
162 self.assertEqual(file.updatedExtension(".txt"), file)
163 self.assertEqual(id(file.updatedExtension(".txt")), id(file))
165 fits = file.updatedExtension(".fits.gz")
166 self.assertEqual(fits.basename(), "test.fits.gz")
167 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
169 def testRelative(self):
170 """Check that we can get subpaths back from two URIs"""
171 parent = ResourcePath(self.tmpdir, forceDirectory=True, forceAbsolute=True)
172 self.assertTrue(parent.isdir())
173 child = ResourcePath(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True)
175 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
177 not_child = ResourcePath("/a/b/dir1/file.txt")
178 self.assertIsNone(not_child.relative_to(parent))
179 self.assertFalse(not_child.isdir())
181 not_directory = ResourcePath(os.path.join(self.tmpdir, "dir1", "file2.txt"))
182 self.assertIsNone(child.relative_to(not_directory))
184 # Relative URIs
185 parent = ResourcePath("a/b/", forceAbsolute=False)
186 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
187 self.assertFalse(child.scheme)
188 self.assertEqual(child.relative_to(parent), "c/d.txt")
190 # forceAbsolute=True should work even on an existing ResourcePath
191 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
193 # File URI and schemeless URI
194 parent = ResourcePath("file:/a/b/c/")
195 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
197 # If the child is relative and the parent is absolute we assume
198 # that the child is a child of the parent unless it uses ".."
199 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
201 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
202 self.assertIsNone(child.relative_to(parent))
204 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
205 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
207 # Test non-file root with relative path.
208 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
209 parent = ResourcePath("s3://hello/a/b/c/")
210 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
212 # Test with different netloc
213 child = ResourcePath("http://my.host/a/b/c.txt")
214 parent = ResourcePath("http://other.host/a/")
215 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
217 # Schemeless absolute child.
218 # Schemeless absolute URI is constructed using root= parameter.
219 parent = ResourcePath("file:///a/b/c/")
220 child = ResourcePath("d/e.txt", root=parent)
221 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
223 parent = ResourcePath("c/", root="/a/b/")
224 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
226 # Absolute schemeless child with relative parent will always fail.
227 parent = ResourcePath("d/e.txt", forceAbsolute=False)
228 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
230 def testParents(self):
231 """Test of splitting and parent walking."""
232 parent = ResourcePath(self.tmpdir, forceDirectory=True, forceAbsolute=True)
233 child_file = parent.join("subdir/file.txt")
234 self.assertFalse(child_file.isdir())
235 child_subdir, file = child_file.split()
236 self.assertEqual(file, "file.txt")
237 self.assertTrue(child_subdir.isdir())
238 self.assertEqual(child_file.dirname(), child_subdir)
239 self.assertEqual(child_file.basename(), file)
240 self.assertEqual(child_file.parent(), child_subdir)
241 derived_parent = child_subdir.parent()
242 self.assertEqual(derived_parent, parent)
243 self.assertTrue(derived_parent.isdir())
244 self.assertEqual(child_file.parent().parent(), parent)
246 def testEnvVar(self):
247 """Test that environment variables are expanded."""
249 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}):
250 uri = ResourcePath("${MY_TEST_DIR}/d.txt")
251 self.assertEqual(uri.path, "/a/b/c/d.txt")
252 self.assertEqual(uri.scheme, "file")
254 # This will not expand
255 uri = ResourcePath("${MY_TEST_DIR}/d.txt", forceAbsolute=False)
256 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt")
257 self.assertFalse(uri.scheme)
259 def testMkdir(self):
260 tmpdir = ResourcePath(self.tmpdir)
261 newdir = tmpdir.join("newdir/seconddir")
262 newdir.mkdir()
263 self.assertTrue(newdir.exists())
264 newfile = newdir.join("temp.txt")
265 newfile.write("Data".encode())
266 self.assertTrue(newfile.exists())
268 def testTransfer(self):
269 src = ResourcePath(os.path.join(self.tmpdir, "test.txt"))
270 content = "Content is some content\nwith something to say\n\n"
271 src.write(content.encode())
273 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"):
274 dest = ResourcePath(os.path.join(self.tmpdir, f"dest_{mode}.txt"))
275 dest.transfer_from(src, transfer=mode)
276 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
278 with open(dest.ospath, "r") as fh:
279 new_content = fh.read()
280 self.assertEqual(new_content, content)
282 if mode in ("symlink", "relsymlink"):
283 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
285 # If the source and destination are hardlinks of each other
286 # the transfer should work even if overwrite=False.
287 if mode in ("link", "hardlink"):
288 dest.transfer_from(src, transfer=mode)
289 else:
290 with self.assertRaises(
291 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
292 ):
293 dest.transfer_from(src, transfer=mode)
295 dest.transfer_from(src, transfer=mode, overwrite=True)
297 os.remove(dest.ospath)
299 b = src.read()
300 self.assertEqual(b.decode(), new_content)
302 nbytes = 10
303 subset = src.read(size=nbytes)
304 self.assertEqual(len(subset), nbytes)
305 self.assertEqual(subset.decode(), content[:nbytes])
307 with self.assertRaises(ValueError):
308 src.transfer_from(src, transfer="unknown")
310 def testTransferIdentical(self):
311 """Test overwrite of identical files."""
312 dir1 = ResourcePath(os.path.join(self.tmpdir, "dir1"), forceDirectory=True)
313 dir1.mkdir()
314 dir2 = os.path.join(self.tmpdir, "dir2")
315 os.symlink(dir1.ospath, dir2)
317 # Write a test file.
318 src_file = dir1.join("test.txt")
319 content = "0123456"
320 src_file.write(content.encode())
322 # Construct URI to destination that should be identical.
323 dest_file = ResourcePath(os.path.join(dir2), forceDirectory=True).join("test.txt")
324 self.assertTrue(dest_file.exists())
325 self.assertNotEqual(src_file, dest_file)
327 # Transfer it over itself.
328 dest_file.transfer_from(src_file, transfer="symlink", overwrite=True)
329 new_content = dest_file.read().decode()
330 self.assertEqual(content, new_content)
332 def testResource(self):
333 # No resources in this package so need a resource in the main
334 # python distribution.
335 u = ResourcePath("resource://idlelib/Icons/README.txt")
336 self.assertTrue(u.exists(), f"Check {u} exists")
338 content = u.read().decode()
339 self.assertIn("IDLE", content)
341 truncated = u.read(size=9).decode()
342 self.assertEqual(truncated, content[:9])
344 d = ResourcePath("resource://idlelib/Icons", forceDirectory=True)
345 self.assertTrue(u.exists(), f"Check directory {d} exists")
347 j = d.join("README.txt")
348 self.assertEqual(u, j)
349 self.assertFalse(j.dirLike)
350 self.assertFalse(j.isdir())
351 not_there = d.join("not-there.yaml")
352 self.assertFalse(not_there.exists())
354 bad = ResourcePath("resource://bad.module/not.yaml")
355 multi = ResourcePath.mexists([u, bad, not_there])
356 self.assertTrue(multi[u])
357 self.assertFalse(multi[bad])
358 self.assertFalse(multi[not_there])
360 def testEscapes(self):
361 """Special characters in file paths"""
362 src = ResourcePath("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True)
363 self.assertFalse(src.scheme)
364 src.write(b"Some content")
365 self.assertTrue(src.exists())
367 # abspath always returns a file scheme
368 file = src.abspath()
369 self.assertTrue(file.exists())
370 self.assertIn("???", file.ospath)
371 self.assertNotIn("???", file.path)
373 file = file.updatedFile("tests??.txt")
374 self.assertNotIn("??.txt", file.path)
375 file.write(b"Other content")
376 self.assertEqual(file.read(), b"Other content")
378 src = src.updatedFile("tests??.txt")
379 self.assertIn("??.txt", src.path)
380 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}")
382 # File URI and schemeless URI
383 parent = ResourcePath("file:" + urllib.parse.quote("/a/b/c/de/??/"))
384 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
385 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
387 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
388 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
390 child = ResourcePath("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))
391 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
393 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
395 # Schemeless so should not quote
396 dir = ResourcePath("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True)
397 self.assertIn("???", dir.ospath)
398 self.assertIn("???", dir.path)
399 self.assertFalse(dir.scheme)
401 # dir.join() morphs into a file scheme
402 new = dir.join("test_j.txt")
403 self.assertIn("???", new.ospath, f"Checking {new}")
404 new.write(b"Content")
406 new2name = "###/test??.txt"
407 new2 = dir.join(new2name)
408 self.assertIn("???", new2.ospath)
409 new2.write(b"Content")
410 self.assertTrue(new2.ospath.endswith(new2name))
411 self.assertEqual(new.read(), new2.read())
413 fdir = dir.abspath()
414 self.assertNotIn("???", fdir.path)
415 self.assertIn("???", fdir.ospath)
416 self.assertEqual(fdir.scheme, "file")
417 fnew = dir.join("test_jf.txt")
418 fnew.write(b"Content")
420 fnew2 = fdir.join(new2name)
421 fnew2.write(b"Content")
422 self.assertTrue(fnew2.ospath.endswith(new2name))
423 self.assertNotIn("###", fnew2.path)
425 self.assertEqual(fnew.read(), fnew2.read())
427 # Test that children relative to schemeless and file schemes
428 # still return the same unquoted name
429 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
430 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
431 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
432 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
434 # Check for double quoting
435 plus_path = "/a/b/c+d/"
436 with self.assertLogs(level="WARNING"):
437 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
438 self.assertEqual(uri.ospath, plus_path)
440 # Check that # is not escaped for schemeless URIs
441 hash_path = "/a/b#/c&d#xyz"
442 hpos = hash_path.rfind("#")
443 uri = ResourcePath(hash_path)
444 self.assertEqual(uri.ospath, hash_path[:hpos])
445 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
447 def testHash(self):
448 """Test that we can store URIs in sets and as keys."""
449 uri1 = ResourcePath(TESTDIR)
450 uri2 = uri1.join("test/")
451 s = {uri1, uri2}
452 self.assertIn(uri1, s)
454 d = {uri1: "1", uri2: "2"}
455 self.assertEqual(d[uri2], "2")
457 def testWalk(self):
458 """Test ResourcePath.walk()."""
459 test_dir_uri = ResourcePath(TESTDIR)
461 # Look for a file that is not there
462 file = test_dir_uri.join("config/basic/butler.yaml")
463 found = list(ResourcePath.findFileResources([file]))
464 self.assertEqual(found[0], file)
466 # Compare against the full local paths
467 expected = set(
468 p for p in glob.glob(os.path.join(TESTDIR, "data", "**"), recursive=True) if os.path.isfile(p)
469 )
470 found = set(u.ospath for u in ResourcePath.findFileResources([test_dir_uri.join("data")]))
471 self.assertEqual(found, expected)
473 # Now solely the YAML files
474 expected_yaml = set(glob.glob(os.path.join(TESTDIR, "data", "**", "*.yaml"), recursive=True))
475 found = set(
476 u.ospath
477 for u in ResourcePath.findFileResources([test_dir_uri.join("data")], file_filter=r".*\.yaml$")
478 )
479 self.assertEqual(found, expected_yaml)
481 # Now two explicit directories and a file
482 expected = set(glob.glob(os.path.join(TESTDIR, "data", "dir1", "*.yaml"), recursive=True))
483 expected.update(set(glob.glob(os.path.join(TESTDIR, "data", "dir2", "*.yaml"), recursive=True)))
484 expected.add(file.ospath)
486 found = set(
487 u.ospath
488 for u in ResourcePath.findFileResources(
489 [file, test_dir_uri.join("data/dir1"), test_dir_uri.join("data/dir2")],
490 file_filter=r".*\.yaml$",
491 )
492 )
493 self.assertEqual(found, expected)
495 # Group by directory -- find everything and compare it with what
496 # we expected to be there in total.
497 found_yaml = set()
498 counter = 0
499 for uris in ResourcePath.findFileResources(
500 [file, test_dir_uri.join("data/")], file_filter=r".*\.yaml$", grouped=True
501 ):
502 found = set(u.ospath for u in uris)
503 if found:
504 counter += 1
506 found_yaml.update(found)
508 expected_yaml_2 = expected_yaml
509 expected_yaml_2.add(file.ospath)
510 self.assertEqual(found_yaml, expected_yaml)
511 self.assertEqual(counter, 3)
513 # Grouping but check that single files are returned in a single group
514 # at the end
515 file2 = test_dir_uri.join("config/templates/templates-bad.yaml")
516 found = list(
517 ResourcePath.findFileResources([file, file2, test_dir_uri.join("data/dir2")], grouped=True)
518 )
519 self.assertEqual(len(found), 2)
520 self.assertEqual(list(found[1]), [file, file2])
522 with self.assertRaises(ValueError):
523 list(file.walk())
525 def testRootURI(self):
526 """Test ResourcePath.root_uri()."""
527 uri = ResourcePath("https://www.notexist.com:8080/file/test")
528 uri2 = ResourcePath("s3://www.notexist.com/file/test")
529 self.assertEqual(uri.root_uri().geturl(), "https://www.notexist.com:8080/")
530 self.assertEqual(uri2.root_uri().geturl(), "s3://www.notexist.com/")
532 def testJoin(self):
533 """Test .join method."""
535 root_str = "s3://bucket/hsc/payload/"
536 root = ResourcePath(root_str)
538 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
539 add_dir = root.join("b/c/d/")
540 self.assertTrue(add_dir.isdir())
541 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
543 up_relative = root.join("../b/c.txt")
544 self.assertFalse(up_relative.isdir())
545 self.assertEqual(up_relative.geturl(), "s3://bucket/hsc/b/c.txt")
547 quote_example = "b&c.t@x#t"
548 needs_quote = root.join(quote_example)
549 self.assertEqual(needs_quote.unquoted_path, f"/hsc/payload/{quote_example}")
551 other = ResourcePath("file://localhost/test.txt")
552 self.assertEqual(root.join(other), other)
553 self.assertEqual(other.join("b/new.txt").geturl(), "file://localhost/b/new.txt")
555 joined = ResourcePath("s3://bucket/hsc/payload/").join(
556 ResourcePath("test.qgraph", forceAbsolute=False)
557 )
558 self.assertEqual(joined, ResourcePath("s3://bucket/hsc/payload/test.qgraph"))
560 with self.assertRaises(ValueError):
561 ResourcePath("s3://bucket/hsc/payload/").join(ResourcePath("test.qgraph"))
563 def testTemporary(self):
564 with ResourcePath.temporary_uri(suffix=".json") as tmp:
565 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
566 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
567 self.assertFalse(tmp.exists(), f"uri: {tmp}")
568 tmp.write(b"abcd")
569 self.assertTrue(tmp.exists(), f"uri: {tmp}")
570 self.assertTrue(tmp.isTemporary)
571 self.assertFalse(tmp.exists(), f"uri: {tmp}")
573 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
574 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".yaml") as tmp:
575 # Use a specified tmpdir and check it is okay for the file
576 # to not be created.
577 self.assertFalse(tmp.exists(), f"uri: {tmp}")
578 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
580 def test_open(self):
581 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
582 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
583 _check_open(self, tmp, mode_suffixes=("", "t"))
584 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
585 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
586 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
587 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
588 _check_open(self, tmp, mode_suffixes=("b",))
589 _check_open(self, tmp, mode_suffixes=("b"), prefer_file_temporary=True)
592@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
593@mock_s3
594class S3URITestCase(unittest.TestCase):
595 """Tests involving S3"""
597 bucketName = "any_bucket"
598 """Bucket name to use in tests"""
600 def setUp(self):
601 # Local test directory
602 self.tmpdir = makeTestTempDir(TESTDIR)
604 # set up some fake credentials if they do not exist
605 self.usingDummyCredentials = setAwsEnvCredentials()
607 # MOTO needs to know that we expect Bucket bucketname to exist
608 s3 = boto3.resource("s3")
609 s3.create_bucket(Bucket=self.bucketName)
611 def tearDown(self):
612 s3 = boto3.resource("s3")
613 bucket = s3.Bucket(self.bucketName)
614 try:
615 bucket.objects.all().delete()
616 except botocore.exceptions.ClientError as e:
617 if e.response["Error"]["Code"] == "404":
618 # the key was not reachable - pass
619 pass
620 else:
621 raise
623 bucket = s3.Bucket(self.bucketName)
624 bucket.delete()
626 # unset any potentially set dummy credentials
627 if self.usingDummyCredentials:
628 unsetAwsEnvCredentials()
630 shutil.rmtree(self.tmpdir, ignore_errors=True)
632 def makeS3Uri(self, path):
633 return f"s3://{self.bucketName}/{path}"
635 def testTransfer(self):
636 src = ResourcePath(os.path.join(self.tmpdir, "test.txt"))
637 content = "Content is some content\nwith something to say\n\n"
638 src.write(content.encode())
639 self.assertTrue(src.exists())
640 self.assertEqual(src.size(), len(content.encode()))
642 dest = ResourcePath(self.makeS3Uri("test.txt"))
643 self.assertFalse(dest.exists())
645 with self.assertRaises(FileNotFoundError):
646 dest.size()
648 dest.transfer_from(src, transfer="copy")
649 self.assertTrue(dest.exists())
651 dest2 = ResourcePath(self.makeS3Uri("copied.txt"))
652 dest2.transfer_from(dest, transfer="copy")
653 self.assertTrue(dest2.exists())
655 local = ResourcePath(os.path.join(self.tmpdir, "copied.txt"))
656 local.transfer_from(dest2, transfer="copy")
657 with open(local.ospath, "r") as fd:
658 new_content = fd.read()
659 self.assertEqual(new_content, content)
661 with self.assertRaises(ValueError):
662 dest2.transfer_from(local, transfer="symlink")
664 b = dest.read()
665 self.assertEqual(b.decode(), new_content)
667 nbytes = 10
668 subset = dest.read(size=nbytes)
669 self.assertEqual(len(subset), nbytes) # Extra byte comes back
670 self.assertEqual(subset.decode(), content[:nbytes])
672 with self.assertRaises(FileExistsError):
673 dest.transfer_from(src, transfer="copy")
675 dest.transfer_from(src, transfer="copy", overwrite=True)
677 def testWalk(self):
678 """Test that we can list an S3 bucket"""
679 # Files we want to create
680 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json")
681 expected_uris = [ResourcePath(self.makeS3Uri(path)) for path in expected]
682 for uri in expected_uris:
683 # Doesn't matter what we write
684 uri.write("123".encode())
686 # Find all the files in the a/ tree
687 found = set(uri.path for uri in ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("a/"))]))
688 self.assertEqual(found, {uri.path for uri in expected_uris})
690 # Find all the files in the a/ tree but group by folder
691 found = ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("a/"))], grouped=True)
692 expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt",), ("/a/b/c/d/v.json",))
694 for got, expect in zip(found, expected):
695 self.assertEqual(tuple(u.path for u in got), expect)
697 # Find only JSON files
698 found = set(
699 uri.path
700 for uri in ResourcePath.findFileResources(
701 [ResourcePath(self.makeS3Uri("a/"))], file_filter=r"\.json$"
702 )
703 )
704 self.assertEqual(found, {uri.path for uri in expected_uris if uri.path.endswith(".json")})
706 # JSON files grouped by directory
707 found = ResourcePath.findFileResources(
708 [ResourcePath(self.makeS3Uri("a/"))], file_filter=r"\.json$", grouped=True
709 )
710 expected = (("/a/z.json",), ("/a/b/c/d/v.json",))
712 for got, expect in zip(found, expected):
713 self.assertEqual(tuple(u.path for u in got), expect)
715 # Check pagination works with large numbers of files. S3 API limits
716 # us to 1000 response per list_objects call so create lots of files
717 created = set()
718 counter = 1
719 n_dir1 = 1100
720 while counter <= n_dir1:
721 new = ResourcePath(self.makeS3Uri(f"test/file{counter:04d}.txt"))
722 new.write(f"{counter}".encode())
723 created.add(str(new))
724 counter += 1
725 counter = 1
726 # Put some in a subdirectory to make sure we are looking in a
727 # hierarchy.
728 n_dir2 = 100
729 while counter <= n_dir2:
730 new = ResourcePath(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt"))
731 new.write(f"{counter}".encode())
732 created.add(str(new))
733 counter += 1
735 found = ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("test/"))])
736 self.assertEqual({str(u) for u in found}, created)
738 # Again with grouping.
739 found = list(ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("test/"))], grouped=True))
740 self.assertEqual(len(found), 2)
741 dir_1 = list(found[0])
742 dir_2 = list(found[1])
743 self.assertEqual(len(dir_1), n_dir1)
744 self.assertEqual(len(dir_2), n_dir2)
746 def testWrite(self):
747 s3write = ResourcePath(self.makeS3Uri("created.txt"))
748 content = "abcdefghijklmnopqrstuv\n"
749 s3write.write(content.encode())
750 self.assertEqual(s3write.read().decode(), content)
752 def testTemporary(self):
753 s3root = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True)
754 with ResourcePath.temporary_uri(prefix=s3root, suffix=".json") as tmp:
755 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
756 self.assertEqual(tmp.scheme, "s3", f"uri: {tmp}")
757 self.assertEqual(tmp.parent(), s3root)
758 basename = tmp.basename()
759 content = "abcd"
760 tmp.write(content.encode())
761 self.assertTrue(tmp.exists(), f"uri: {tmp}")
762 self.assertFalse(tmp.exists())
764 # Again without writing anything, to check that there is no complaint
765 # on exit of context manager.
766 with ResourcePath.temporary_uri(prefix=s3root, suffix=".json") as tmp:
767 self.assertFalse(tmp.exists())
768 # Check that the file has a different name than before.
769 self.assertNotEqual(tmp.basename(), basename, f"uri: {tmp}")
770 self.assertFalse(tmp.exists())
772 def testRelative(self):
773 """Check that we can get subpaths back from two URIs"""
774 parent = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True)
775 child = ResourcePath(self.makeS3Uri("rootdir/dir1/file.txt"))
777 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
779 not_child = ResourcePath(self.makeS3Uri("/a/b/dir1/file.txt"))
780 self.assertFalse(not_child.relative_to(parent))
782 not_s3 = ResourcePath(os.path.join(self.tmpdir, "dir1", "file2.txt"))
783 self.assertFalse(child.relative_to(not_s3))
785 def testQuoting(self):
786 """Check that quoting works."""
787 parent = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True)
788 subpath = "rootdir/dir1+/file?.txt"
789 child = ResourcePath(self.makeS3Uri(urllib.parse.quote(subpath)))
791 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
792 self.assertEqual(child.basename(), "file?.txt")
793 self.assertEqual(child.relativeToPathRoot, subpath)
794 self.assertIn("%", child.path)
795 self.assertEqual(child.unquoted_path, "/" + subpath)
797 def test_open(self):
798 text_uri = ResourcePath(self.makeS3Uri("file.txt"))
799 _check_open(self, text_uri, mode_suffixes=("", "t"))
800 _check_open(self, text_uri, mode_suffixes=("t",), encoding="utf-16")
801 _check_open(self, text_uri, mode_suffixes=("t",), prefer_file_temporary=True)
802 _check_open(self, text_uri, mode_suffixes=("t",), prefer_file_temporary=True, encoding="utf-16")
803 binary_uri = ResourcePath(self.makeS3Uri("file.dat"))
804 _check_open(self, binary_uri, mode_suffixes=("b",))
805 _check_open(self, binary_uri, mode_suffixes=("b",), prefer_file_temporary=True)
808# Mock required environment variables during tests
809@unittest.mock.patch.dict(
810 os.environ,
811 {
812 "LSST_BUTLER_WEBDAV_AUTH": "TOKEN",
813 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join(TESTDIR, "data/webdav/token"),
814 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs",
815 },
816)
817class WebdavURITestCase(unittest.TestCase):
818 def setUp(self):
819 # Local test directory
820 self.tmpdir = makeTestTempDir(TESTDIR)
822 serverRoot = "www.not-exists.orgx"
823 existingFolderName = "existingFolder"
824 existingFileName = "existingFile"
825 notExistingFileName = "notExistingFile"
827 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True)
828 self.existingFileResourcePath = ResourcePath(
829 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
830 )
831 self.notExistingFileResourcePath = ResourcePath(
832 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
833 )
834 self.existingFolderResourcePath = ResourcePath(
835 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
836 )
837 self.notExistingFolderResourcePath = ResourcePath(
838 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
839 )
841 # Need to declare the options
842 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"})
844 # Used by HttpResourcePath.exists()
845 responses.add(
846 responses.HEAD,
847 self.existingFileResourcePath.geturl(),
848 status=200,
849 headers={"Content-Length": "1024"},
850 )
851 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404)
853 # Used by HttpResourcePath.read()
854 responses.add(
855 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!")
856 )
857 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404)
859 # Used by HttpResourcePath.write()
860 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201)
862 # Used by HttpResourcePath.transfer_from()
863 responses.add(
864 responses.Response(
865 url=self.existingFileResourcePath.geturl(),
866 method="COPY",
867 headers={"Destination": self.existingFileResourcePath.geturl()},
868 status=201,
869 )
870 )
871 responses.add(
872 responses.Response(
873 url=self.existingFileResourcePath.geturl(),
874 method="COPY",
875 headers={"Destination": self.notExistingFileResourcePath.geturl()},
876 status=201,
877 )
878 )
879 responses.add(
880 responses.Response(
881 url=self.existingFileResourcePath.geturl(),
882 method="MOVE",
883 headers={"Destination": self.notExistingFileResourcePath.geturl()},
884 status=201,
885 )
886 )
888 # Used by HttpResourcePath.remove()
889 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200)
890 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404)
892 # Used by HttpResourcePath.mkdir()
893 responses.add(
894 responses.HEAD,
895 self.existingFolderResourcePath.geturl(),
896 status=200,
897 headers={"Content-Length": "1024"},
898 )
899 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"})
900 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404)
901 responses.add(
902 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201)
903 )
904 responses.add(
905 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403)
906 )
908 @responses.activate
909 def testExists(self):
911 self.assertTrue(self.existingFileResourcePath.exists())
912 self.assertFalse(self.notExistingFileResourcePath.exists())
914 self.assertEqual(self.existingFileResourcePath.size(), 1024)
915 with self.assertRaises(FileNotFoundError):
916 self.notExistingFileResourcePath.size()
918 @responses.activate
919 def testRemove(self):
921 self.assertIsNone(self.existingFileResourcePath.remove())
922 with self.assertRaises(FileNotFoundError):
923 self.notExistingFileResourcePath.remove()
925 @responses.activate
926 def testMkdir(self):
928 # The mock means that we can't check this now exists
929 self.notExistingFolderResourcePath.mkdir()
931 # This should do nothing
932 self.existingFolderResourcePath.mkdir()
934 with self.assertRaises(ValueError):
935 self.notExistingFileResourcePath.mkdir()
937 @responses.activate
938 def testRead(self):
940 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!")
941 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.")
942 with self.assertRaises(FileNotFoundError):
943 self.notExistingFileResourcePath.read()
945 # Run this twice to ensure use of cache in code coverag.
946 for _ in (1, 2):
947 with self.existingFileResourcePath.as_local() as local_uri:
948 self.assertTrue(local_uri.isLocal)
949 content = local_uri.read().decode()
950 self.assertEqual(content, "It works!")
952 # Check that the environment variable is being read.
953 lsst.resources.http._TMPDIR = None
954 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir}):
955 with self.existingFileResourcePath.as_local() as local_uri:
956 self.assertTrue(local_uri.isLocal)
957 content = local_uri.read().decode()
958 self.assertEqual(content, "It works!")
959 self.assertIsNotNone(local_uri.relative_to(ResourcePath(self.tmpdir)))
961 @responses.activate
962 def testWrite(self):
964 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content.")))
965 with self.assertRaises(FileExistsError):
966 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False)
968 @responses.activate
969 def testTransfer(self):
971 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath))
972 self.assertIsNone(
973 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move")
974 )
975 with self.assertRaises(FileExistsError):
976 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
977 with self.assertRaises(ValueError):
978 self.notExistingFileResourcePath.transfer_from(
979 src=self.existingFileResourcePath, transfer="unsupported"
980 )
982 def testParent(self):
984 self.assertEqual(
985 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl()
986 )
987 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
988 self.assertEqual(
989 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl()
990 )
993if __name__ == "__main__": 993 ↛ 994line 993 didn't jump to line 994, because the condition on line 993 was never true
994 unittest.main()