Coverage for python/lsst/resources/tests.py: 9%
498 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-09 09:05 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-09 09:05 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from typing import Any, Callable, Iterable, Optional, Union
25from lsst.resources import ResourcePath
26from lsst.resources.utils import makeTestTempDir, removeTestTempDir
28TESTDIR = os.path.abspath(os.path.dirname(__file__))
31def _check_open(
32 test_case: Union[_GenericTestCase, unittest.TestCase],
33 uri: ResourcePath,
34 *,
35 mode_suffixes: Iterable[str] = ("", "t", "b"),
36 **kwargs: Any,
37) -> None:
38 """Test an implementation of ButlerURI.open.
40 Parameters
41 ----------
42 test_case : `unittest.TestCase`
43 Test case to use for assertions.
44 uri : `ResourcePath`
45 URI to use for tests. Must point to a writeable location that is not
46 yet occupied by a file. On return, the location may point to a file
47 only if the test fails.
48 mode_suffixes : `Iterable` of `str`
49 Suffixes to pass as part of the ``mode`` argument to
50 `ResourcePath.open`, indicating whether to open as binary or as text;
51 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
52 **kwargs
53 Additional keyword arguments to forward to all calls to `open`.
54 """
55 text_content = "wxyz🙂"
56 bytes_content = uuid.uuid4().bytes
57 content_by_mode_suffix = {
58 "": text_content,
59 "t": text_content,
60 "b": bytes_content,
61 }
62 empty_content_by_mode_suffix = {
63 "": "",
64 "t": "",
65 "b": b"",
66 }
67 # To appease mypy
68 double_content_by_mode_suffix = {
69 "": text_content + text_content,
70 "t": text_content + text_content,
71 "b": bytes_content + bytes_content,
72 }
73 for mode_suffix in mode_suffixes:
74 content = content_by_mode_suffix[mode_suffix]
75 double_content = double_content_by_mode_suffix[mode_suffix]
76 # Create file with mode='x', which prohibits overwriting.
77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
78 write_buffer.write(content)
79 test_case.assertTrue(uri.exists())
80 # Check that opening with 'x' now raises, and does not modify content.
81 with test_case.assertRaises(FileExistsError):
82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
83 write_buffer.write("bad")
84 # Read the file we created and check the contents.
85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
86 test_case.assertEqual(read_buffer.read(), content)
87 # Write two copies of the content, overwriting the single copy there.
88 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
89 write_buffer.write(double_content)
90 # Read again, this time use mode='r+', which reads what is there and
91 # then lets us write more; we'll use that to reset the file to one
92 # copy of the content.
93 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
94 test_case.assertEqual(rw_buffer.read(), double_content)
95 rw_buffer.seek(0)
96 rw_buffer.truncate()
97 rw_buffer.write(content)
98 rw_buffer.seek(0)
99 test_case.assertEqual(rw_buffer.read(), content)
100 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
101 test_case.assertEqual(read_buffer.read(), content)
102 # Append some more content to the file; should now have two copies.
103 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
104 append_buffer.write(content)
105 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
106 test_case.assertEqual(read_buffer.read(), double_content)
107 # Final mode to check is w+, which does read/write but truncates first.
108 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
109 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
110 rw_buffer.write(content)
111 rw_buffer.seek(0)
112 test_case.assertEqual(rw_buffer.read(), content)
113 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
114 test_case.assertEqual(read_buffer.read(), content)
115 # Remove file to make room for the next loop of tests with this URI.
116 uri.remove()
119class _GenericTestCase:
120 """Generic base class for test mixin."""
122 scheme: Optional[str] = None
123 netloc: Optional[str] = None
124 base_path: Optional[str] = None
125 path1 = "test_dir"
126 path2 = "file.txt"
128 # Because we use a mixin for tests mypy needs to understand that
129 # the unittest.TestCase methods exist.
130 # We do not inherit from unittest.TestCase because that results
131 # in the tests defined here being run as well as the tests in the
132 # test file itself. We can make those tests skip but it gives an
133 # uniformative view of how many tests are running.
134 assertEqual: Callable
135 assertNotEqual: Callable
136 assertIsNone: Callable
137 assertIn: Callable
138 assertNotIn: Callable
139 assertFalse: Callable
140 assertTrue: Callable
141 assertRaises: Callable
142 assertLogs: Callable
144 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str:
145 if self.scheme is not None:
146 if netloc is None:
147 netloc = self.netloc
148 if path.startswith("/"):
149 path = path[1:]
150 if self.base_path is not None:
151 path = f"{self.base_path}/{path}".lstrip("/")
153 return f"{self.scheme}://{netloc}/{path}"
154 else:
155 return path
158class GenericTestCase(_GenericTestCase):
159 """Test cases for generic manipulation of a `ResourcePath`"""
161 def setUp(self) -> None:
162 if self.scheme is None:
163 raise unittest.SkipTest("No scheme defined")
164 self.root = self._make_uri("")
165 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
167 def test_creation(self) -> None:
168 self.assertEqual(self.root_uri.scheme, self.scheme)
169 self.assertEqual(self.root_uri.netloc, self.netloc)
170 self.assertFalse(self.root_uri.query)
171 self.assertFalse(self.root_uri.params)
173 with self.assertRaises(ValueError):
174 ResourcePath({}) # type: ignore
176 with self.assertRaises(RuntimeError):
177 ResourcePath(self.root_uri, isTemporary=True)
179 file = self.root_uri.join("file.txt")
180 with self.assertRaises(RuntimeError):
181 ResourcePath(file, forceDirectory=True)
183 with self.assertRaises(NotImplementedError):
184 ResourcePath("unknown://netloc")
186 replaced = file.replace(fragment="frag")
187 self.assertEqual(replaced.fragment, "frag")
189 with self.assertRaises(ValueError):
190 file.replace(scheme="new")
192 self.assertNotEqual(replaced, str(replaced))
193 self.assertNotEqual(str(replaced), replaced)
195 def test_extension(self) -> None:
196 uri = ResourcePath(self._make_uri("dir/test.txt"))
197 self.assertEqual(uri.updatedExtension(None), uri)
198 self.assertEqual(uri.updatedExtension(".txt"), uri)
199 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
201 fits = uri.updatedExtension(".fits.gz")
202 self.assertEqual(fits.basename(), "test.fits.gz")
203 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
205 extensionless = self.root_uri.join("no_ext")
206 self.assertEqual(extensionless.getExtension(), "")
207 extension = extensionless.updatedExtension(".fits")
208 self.assertEqual(extension.getExtension(), ".fits")
210 def test_relative(self) -> None:
211 """Check that we can get subpaths back from two URIs"""
212 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
213 self.assertTrue(parent.isdir())
214 child = parent.join("dir1/file.txt")
216 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
218 not_child = ResourcePath("/a/b/dir1/file.txt")
219 self.assertIsNone(not_child.relative_to(parent))
220 self.assertFalse(not_child.isdir())
222 not_directory = parent.join("dir1/file2.txt")
223 self.assertIsNone(child.relative_to(not_directory))
225 # Relative URIs
226 parent = ResourcePath("a/b/", forceAbsolute=False)
227 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
228 self.assertFalse(child.scheme)
229 self.assertEqual(child.relative_to(parent), "c/d.txt")
231 # forceAbsolute=True should work even on an existing ResourcePath
232 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
234 # Absolute URI and schemeless URI
235 parent = self.root_uri.join("/a/b/c/")
236 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
238 # If the child is relative and the parent is absolute we assume
239 # that the child is a child of the parent unless it uses ".."
240 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
242 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
243 self.assertIsNone(child.relative_to(parent))
245 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
246 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
248 # Test with different netloc
249 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
250 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True)
251 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
253 # This is an absolute path so will *always* return a file URI and
254 # ignore the root parameter.
255 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True)
256 self.assertEqual(parent.geturl(), "file:///a/b/c/")
258 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True)
259 child = ResourcePath("d/e.txt", root=parent)
260 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
262 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/")))
263 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
265 # Absolute schemeless child with relative parent will always fail.
266 child = ResourcePath("d/e.txt", root="/a/b/c")
267 parent = ResourcePath("d/e.txt", forceAbsolute=False)
268 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
270 def test_parents(self) -> None:
271 """Test of splitting and parent walking."""
272 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
273 child_file = parent.join("subdir/file.txt")
274 self.assertFalse(child_file.isdir())
275 child_subdir, file = child_file.split()
276 self.assertEqual(file, "file.txt")
277 self.assertTrue(child_subdir.isdir())
278 self.assertEqual(child_file.dirname(), child_subdir)
279 self.assertEqual(child_file.basename(), file)
280 self.assertEqual(child_file.parent(), child_subdir)
281 derived_parent = child_subdir.parent()
282 self.assertEqual(derived_parent, parent)
283 self.assertTrue(derived_parent.isdir())
284 self.assertEqual(child_file.parent().parent(), parent)
286 def test_escapes(self) -> None:
287 """Special characters in file paths"""
288 src = self.root_uri.join("bbb/???/test.txt")
289 self.assertNotIn("???", src.path)
290 self.assertIn("???", src.unquoted_path)
292 file = src.updatedFile("tests??.txt")
293 self.assertNotIn("??.txt", file.path)
295 src = src.updatedFile("tests??.txt")
296 self.assertIn("??.txt", src.unquoted_path)
298 # File URI and schemeless URI
299 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
300 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
301 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
303 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
304 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
306 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
307 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
309 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
311 # dir.join() morphs into a file scheme
312 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
313 new = dir.join("test_j.txt")
314 self.assertIn("???", new.unquoted_path, f"Checking {new}")
316 new2name = "###/test??.txt"
317 new2 = dir.join(new2name)
318 self.assertIn("???", new2.unquoted_path)
319 self.assertTrue(new2.unquoted_path.endswith(new2name))
321 fdir = dir.abspath()
322 self.assertNotIn("???", fdir.path)
323 self.assertIn("???", fdir.unquoted_path)
324 self.assertEqual(fdir.scheme, self.scheme)
326 fnew2 = fdir.join(new2name)
327 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
328 self.assertNotIn("###", fnew2.path)
330 # Test that children relative to schemeless and file schemes
331 # still return the same unquoted name
332 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
333 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
334 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
335 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
337 # Check for double quoting
338 plus_path = "/a/b/c+d/"
339 with self.assertLogs(level="WARNING"):
340 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
341 self.assertEqual(uri.ospath, plus_path)
343 # Check that # is not escaped for schemeless URIs
344 hash_path = "/a/b#/c&d#xyz"
345 hpos = hash_path.rfind("#")
346 uri = ResourcePath(hash_path)
347 self.assertEqual(uri.ospath, hash_path[:hpos])
348 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
350 def test_hash(self) -> None:
351 """Test that we can store URIs in sets and as keys."""
352 uri1 = self.root_uri
353 uri2 = uri1.join("test/")
354 s = {uri1, uri2}
355 self.assertIn(uri1, s)
357 d = {uri1: "1", uri2: "2"}
358 self.assertEqual(d[uri2], "2")
360 def test_root_uri(self) -> None:
361 """Test ResourcePath.root_uri()."""
362 uri = ResourcePath(self._make_uri("a/b/c.txt"))
363 self.assertEqual(uri.root_uri().geturl(), self.root)
365 def test_join(self) -> None:
366 """Test .join method."""
367 root_str = self.root
368 root = self.root_uri
370 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
371 add_dir = root.join("b/c/d/")
372 self.assertTrue(add_dir.isdir())
373 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
375 up_relative = root.join("../b/c.txt")
376 self.assertFalse(up_relative.isdir())
377 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
379 quote_example = "hsc/payload/b&c.t@x#t"
380 needs_quote = root.join(quote_example)
381 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
383 other = ResourcePath(f"{self.root}test.txt")
384 self.assertEqual(root.join(other), other)
385 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
387 joined = ResourcePath(f"{self.root}hsc/payload/").join(
388 ResourcePath("test.qgraph", forceAbsolute=False)
389 )
390 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
392 qgraph = ResourcePath("test.qgraph") # Absolute URI
393 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph)
394 self.assertEqual(joined, qgraph)
396 def test_quoting(self) -> None:
397 """Check that quoting works."""
398 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
399 subpath = "rootdir/dir1+/file?.txt"
400 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
402 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
403 self.assertEqual(child.basename(), "file?.txt")
404 self.assertEqual(child.relativeToPathRoot, subpath)
405 self.assertIn("%", child.path)
406 self.assertEqual(child.unquoted_path, "/" + subpath)
408 def test_ordering(self) -> None:
409 """Check that greater/less comparison operators work."""
410 a = self._make_uri("a.txt")
411 b = self._make_uri("b/")
412 self.assertTrue(a < b)
413 self.assertFalse(a < a)
414 self.assertTrue(a <= b)
415 self.assertTrue(a <= a)
416 self.assertTrue(b > a)
417 self.assertFalse(b > b)
418 self.assertTrue(b >= a)
419 self.assertTrue(b >= b)
422class GenericReadWriteTestCase(_GenericTestCase):
423 """Test schemes that can read and write using concrete resources."""
425 transfer_modes = ("copy", "move")
426 testdir: Optional[str] = None
428 def setUp(self) -> None:
429 if self.scheme is None:
430 raise unittest.SkipTest("No scheme defined")
431 self.root = self._make_uri("")
432 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
434 if self.scheme == "file":
435 # Use a local tempdir because on macOS the temp dirs use symlinks
436 # so relsymlink gets quite confused.
437 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
438 else:
439 # Create random tmp directory relative to the test root.
440 self.tmpdir = self.root_uri.join(
441 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
442 forceDirectory=True,
443 )
444 self.tmpdir.mkdir()
446 def tearDown(self) -> None:
447 if self.tmpdir:
448 if self.tmpdir.isLocal:
449 removeTestTempDir(self.tmpdir.ospath)
451 def test_file(self) -> None:
452 uri = self.tmpdir.join("test.txt")
453 self.assertFalse(uri.exists(), f"{uri} should not exist")
454 self.assertTrue(uri.path.endswith("test.txt"))
456 content = "abcdefghijklmnopqrstuv\n"
457 uri.write(content.encode())
458 self.assertTrue(uri.exists(), f"{uri} should now exist")
459 self.assertEqual(uri.read().decode(), content)
460 self.assertEqual(uri.size(), len(content.encode()))
462 with self.assertRaises(FileExistsError):
463 uri.write(b"", overwrite=False)
465 # Not all backends can tell if a remove fails so we can not
466 # test that a remove of a non-existent entry is guaranteed to raise.
467 uri.remove()
468 self.assertFalse(uri.exists())
470 # Ideally the test would remove the file again and raise a
471 # FileNotFoundError. This is not reliable for remote resources
472 # and doing an explicit check before trying to remove the resource
473 # just to raise an exception is deemed an unacceptable overhead.
475 with self.assertRaises(FileNotFoundError):
476 uri.read()
478 with self.assertRaises(FileNotFoundError):
479 self.tmpdir.join("file/not/there.txt").size()
481 # Check that creating a URI from a URI returns the same thing
482 uri2 = ResourcePath(uri)
483 self.assertEqual(uri, uri2)
484 self.assertEqual(id(uri), id(uri2))
486 def test_mkdir(self) -> None:
487 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
488 newdir.mkdir()
489 self.assertTrue(newdir.exists())
490 self.assertEqual(newdir.size(), 0)
492 newfile = newdir.join("temp.txt")
493 newfile.write("Data".encode())
494 self.assertTrue(newfile.exists())
496 file = self.tmpdir.join("file.txt")
497 # Some schemes will realize that the URI is not a file and so
498 # will raise NotADirectoryError. The file scheme is more permissive
499 # and lets you write anything but will raise NotADirectoryError
500 # if a non-directory is already there. We therefore write something
501 # to the file to ensure that we trigger a portable exception.
502 file.write(b"")
503 with self.assertRaises(NotADirectoryError):
504 file.mkdir()
506 # The root should exist.
507 self.root_uri.mkdir()
508 self.assertTrue(self.root_uri.exists())
510 def test_transfer(self) -> None:
511 src = self.tmpdir.join("test.txt")
512 content = "Content is some content\nwith something to say\n\n"
513 src.write(content.encode())
515 can_move = "move" in self.transfer_modes
516 for mode in self.transfer_modes:
517 if mode == "move":
518 continue
520 dest = self.tmpdir.join(f"dest_{mode}.txt")
521 # Ensure that we get some debugging output.
522 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
523 dest.transfer_from(src, transfer=mode)
524 self.assertIn("Transferring ", "\n".join(cm.output))
525 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
527 new_content = dest.read().decode()
528 self.assertEqual(new_content, content)
530 if mode in ("symlink", "relsymlink"):
531 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
533 # If the source and destination are hardlinks of each other
534 # the transfer should work even if overwrite=False.
535 if mode in ("link", "hardlink"):
536 dest.transfer_from(src, transfer=mode)
537 else:
538 with self.assertRaises(
539 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
540 ):
541 dest.transfer_from(src, transfer=mode)
543 # Transfer again and overwrite.
544 dest.transfer_from(src, transfer=mode, overwrite=True)
546 dest.remove()
548 b = src.read()
549 self.assertEqual(b.decode(), new_content)
551 nbytes = 10
552 subset = src.read(size=nbytes)
553 self.assertEqual(len(subset), nbytes)
554 self.assertEqual(subset.decode(), content[:nbytes])
556 # Transferring to self should be okay.
557 src.transfer_from(src, "auto")
559 with self.assertRaises(ValueError):
560 src.transfer_from(src, transfer="unknown")
562 # A move transfer is special.
563 if can_move:
564 dest.transfer_from(src, transfer="move")
565 self.assertFalse(src.exists())
566 self.assertTrue(dest.exists())
567 else:
568 src.remove()
570 dest.remove()
571 with self.assertRaises(FileNotFoundError):
572 dest.transfer_from(src, "auto")
574 def test_local_transfer(self) -> None:
575 """Test we can transfer to and from local file."""
576 remote_src = self.tmpdir.join("src.json")
577 remote_src.write(b"42")
578 remote_dest = self.tmpdir.join("dest.json")
580 with ResourcePath.temporary_uri(suffix=".json") as tmp:
581 self.assertTrue(tmp.isLocal)
582 tmp.transfer_from(remote_src, transfer="auto")
583 self.assertEqual(tmp.read(), remote_src.read())
585 remote_dest.transfer_from(tmp, transfer="auto")
586 self.assertEqual(remote_dest.read(), tmp.read())
588 # Temporary (possibly remote) resource.
589 # Transfers between temporary resources.
590 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
591 # Temporary local resource.
592 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
593 remote_tmp.write(b"42")
594 if not remote_tmp.isLocal:
595 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
596 with self.assertRaises(RuntimeError):
597 # Trying to symlink a remote resource is not going
598 # to work. A hardlink could work but would rely
599 # on the local temp space being on the same
600 # filesystem as the target.
601 local_tmp.transfer_from(remote_tmp, transfer)
602 local_tmp.transfer_from(remote_tmp, "move")
603 self.assertFalse(remote_tmp.exists())
604 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
605 self.assertEqual(local_tmp.read(), remote_tmp.read())
607 # Transfer of missing remote.
608 remote_tmp.remove()
609 with self.assertRaises(FileNotFoundError):
610 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
612 def test_local(self) -> None:
613 """Check that remote resources can be made local."""
614 src = self.tmpdir.join("test.txt")
615 original_content = "Content is some content\nwith something to say\n\n"
616 src.write(original_content.encode())
618 # Run this twice to ensure use of cache in code coverage
619 # if applicable.
620 for _ in (1, 2):
621 with src.as_local() as local_uri:
622 self.assertTrue(local_uri.isLocal)
623 content = local_uri.read().decode()
624 self.assertEqual(content, original_content)
626 if src.isLocal:
627 self.assertEqual(src, local_uri)
629 with self.assertRaises(IsADirectoryError):
630 with self.root_uri.as_local() as local_uri:
631 pass
633 def test_walk(self) -> None:
634 """Walk a directory hierarchy."""
635 root = self.tmpdir.join("walk/")
637 # Look for a file that is not there
638 file = root.join("config/basic/butler.yaml")
639 found_list = list(ResourcePath.findFileResources([file]))
640 self.assertEqual(found_list[0], file)
642 # First create the files (content is irrelevant).
643 expected_files = {
644 "dir1/a.yaml",
645 "dir1/b.yaml",
646 "dir1/c.json",
647 "dir2/d.json",
648 "dir2/e.yaml",
649 }
650 expected_uris = {root.join(f) for f in expected_files}
651 for uri in expected_uris:
652 uri.write(b"")
653 self.assertTrue(uri.exists())
655 # Look for the files.
656 found = set(ResourcePath.findFileResources([root]))
657 self.assertEqual(found, expected_uris)
659 # Now solely the YAML files.
660 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
661 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
662 self.assertEqual(found, expected_yaml)
664 # Now two explicit directories and a file
665 expected = set(u for u in expected_yaml)
666 expected.add(file)
668 found = set(
669 ResourcePath.findFileResources(
670 [file, root.join("dir1/"), root.join("dir2/")],
671 file_filter=r".*\.yaml$",
672 )
673 )
674 self.assertEqual(found, expected)
676 # Group by directory -- find everything and compare it with what
677 # we expected to be there in total.
678 found_yaml = set()
679 counter = 0
680 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
681 assert not isinstance(uris, ResourcePath) # for mypy.
682 found_uris = set(uris)
683 if found_uris:
684 counter += 1
686 found_yaml.update(found_uris)
688 expected_yaml_2 = expected_yaml
689 expected_yaml_2.add(file)
690 self.assertEqual(found_yaml, expected_yaml)
691 self.assertEqual(counter, 3)
693 # Grouping but check that single files are returned in a single group
694 # at the end
695 file2 = root.join("config/templates/templates-bad.yaml")
696 found_grouped = [
697 [uri for uri in group]
698 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
699 if not isinstance(group, ResourcePath) # For mypy.
700 ]
701 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
702 self.assertEqual(list(found_grouped[1]), [file, file2])
704 with self.assertRaises(ValueError):
705 # The list forces the generator to run.
706 list(file.walk())
708 # A directory that does not exist returns nothing.
709 self.assertEqual(list(root.join("dir3/").walk()), [])
711 def test_large_walk(self) -> None:
712 # In some systems pagination is used so ensure that we can handle
713 # large numbers of files. For example S3 limits us to 1000 responses
714 # per listing call.
715 created = set()
716 counter = 1
717 n_dir1 = 1100
718 root = self.tmpdir.join("large_walk", forceDirectory=True)
719 while counter <= n_dir1:
720 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
721 new.write(f"{counter}".encode())
722 created.add(new)
723 counter += 1
724 counter = 1
725 # Put some in a subdirectory to make sure we are looking in a
726 # hierarchy.
727 n_dir2 = 100
728 subdir = root.join("subdir", forceDirectory=True)
729 while counter <= n_dir2:
730 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
731 new.write(f"{counter}".encode())
732 created.add(new)
733 counter += 1
735 found = set(ResourcePath.findFileResources([root]))
736 self.assertEqual(len(found), n_dir1 + n_dir2)
737 self.assertEqual(found, created)
739 # Again with grouping.
740 # (mypy gets upset not knowing which of the two options is being
741 # returned so add useless instance check).
742 found_list = list(
743 [uri for uri in group]
744 for group in ResourcePath.findFileResources([root], grouped=True)
745 if not isinstance(group, ResourcePath) # For mypy.
746 )
747 self.assertEqual(len(found_list), 2)
748 self.assertEqual(len(found_list[0]), n_dir1)
749 self.assertEqual(len(found_list[1]), n_dir2)
751 def test_temporary(self) -> None:
752 prefix = self.tmpdir.join("tmp", forceDirectory=True)
753 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
754 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
755 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
756 self.assertFalse(tmp.exists(), f"uri: {tmp}")
757 tmp.write(b"abcd")
758 self.assertTrue(tmp.exists(), f"uri: {tmp}")
759 self.assertTrue(tmp.isTemporary)
760 self.assertFalse(tmp.exists(), f"uri: {tmp}")
762 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
763 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
764 # Use a specified tmpdir and check it is okay for the file
765 # to not be created.
766 self.assertFalse(tmp.getExtension())
767 self.assertFalse(tmp.exists(), f"uri: {tmp}")
768 self.assertEqual(tmp.scheme, self.scheme)
769 self.assertTrue(tmp.isTemporary)
770 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
772 # Fake a directory suffix.
773 with self.assertRaises(NotImplementedError):
774 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
775 pass
777 def test_open(self) -> None:
778 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
779 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
780 _check_open(self, tmp, mode_suffixes=("", "t"))
781 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
782 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
783 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
784 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
785 _check_open(self, tmp, mode_suffixes=("b",))
786 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
788 with self.assertRaises(IsADirectoryError):
789 with self.root_uri.open():
790 pass
792 def test_mexists(self) -> None:
793 root = self.tmpdir.join("mexists/")
795 # A file that is not there.
796 file = root.join("config/basic/butler.yaml")
798 # Create some files.
799 expected_files = {
800 "dir1/a.yaml",
801 "dir1/b.yaml",
802 "dir2/e.yaml",
803 }
804 expected_uris = {root.join(f) for f in expected_files}
805 for uri in expected_uris:
806 uri.write(b"")
807 self.assertTrue(uri.exists())
808 expected_uris.add(file)
810 multi = ResourcePath.mexists(expected_uris)
812 for uri, is_there in multi.items():
813 if uri == file:
814 self.assertFalse(is_there)
815 else:
816 self.assertTrue(is_there)