Coverage for python/lsst/resources/tests.py: 8%
519 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-19 11:17 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-19 11:17 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from collections.abc import Iterable
24from typing import TYPE_CHECKING, Any
26from lsst.resources import ResourcePath
27from lsst.resources.utils import makeTestTempDir, removeTestTempDir
29TESTDIR = os.path.abspath(os.path.dirname(__file__))
32def _check_open(
33 test_case: _GenericTestCase | unittest.TestCase,
34 uri: ResourcePath,
35 *,
36 mode_suffixes: Iterable[str] = ("", "t", "b"),
37 **kwargs: Any,
38) -> None:
39 """Test an implementation of ButlerURI.open.
41 Parameters
42 ----------
43 test_case : `unittest.TestCase`
44 Test case to use for assertions.
45 uri : `ResourcePath`
46 URI to use for tests. Must point to a writeable location that is not
47 yet occupied by a file. On return, the location may point to a file
48 only if the test fails.
49 mode_suffixes : `~collections.abc.Iterable` of `str`
50 Suffixes to pass as part of the ``mode`` argument to
51 `ResourcePath.open`, indicating whether to open as binary or as text;
52 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
53 **kwargs
54 Additional keyword arguments to forward to all calls to `open`.
55 """
56 text_content = "abcdefghijklmnopqrstuvwxyz🙂"
57 bytes_content = uuid.uuid4().bytes
58 content_by_mode_suffix = {
59 "": text_content,
60 "t": text_content,
61 "b": bytes_content,
62 }
63 empty_content_by_mode_suffix = {
64 "": "",
65 "t": "",
66 "b": b"",
67 }
68 # To appease mypy
69 double_content_by_mode_suffix = {
70 "": text_content + text_content,
71 "t": text_content + text_content,
72 "b": bytes_content + bytes_content,
73 }
74 for mode_suffix in mode_suffixes:
75 content = content_by_mode_suffix[mode_suffix]
76 double_content = double_content_by_mode_suffix[mode_suffix]
77 # Create file with mode='x', which prohibits overwriting.
78 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
79 write_buffer.write(content)
80 test_case.assertTrue(uri.exists())
81 # Check that opening with 'x' now raises, and does not modify content.
82 with test_case.assertRaises(FileExistsError):
83 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
84 write_buffer.write("bad")
85 # Read the file we created and check the contents.
86 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
87 test_case.assertEqual(read_buffer.read(), content)
88 # Check that we can read bytes in a loop and get EOF
89 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
90 # Seek off the end of the file and should read empty back.
91 read_buffer.seek(1024)
92 test_case.assertEqual(read_buffer.tell(), 1024)
93 content_read = read_buffer.read() # Read as much as we can.
94 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
96 # First read more than the content.
97 read_buffer.seek(0)
98 size = len(content) * 3
99 chunk_read = read_buffer.read(size)
100 test_case.assertEqual(chunk_read, content)
102 # Repeated reads should always return empty string.
103 chunk_read = read_buffer.read(size)
104 test_case.assertEqual(len(chunk_read), 0)
105 chunk_read = read_buffer.read(size)
106 test_case.assertEqual(len(chunk_read), 0)
108 # Go back to start of file and read in smaller chunks.
109 read_buffer.seek(0)
110 size = len(content) // 3
112 content_read = empty_content_by_mode_suffix[mode_suffix]
113 n_reads = 0
114 while chunk_read := read_buffer.read(size):
115 content_read += chunk_read
116 n_reads += 1
117 if n_reads > 10: # In case EOF never hits because of bug.
118 raise AssertionError(
119 f"Failed to stop reading from file after {n_reads} loops. "
120 f"Read {len(content_read)} bytes/characters. Expected {len(content)}."
121 )
122 test_case.assertEqual(content_read, content)
124 # Go back to start of file and read the entire thing.
125 read_buffer.seek(0)
126 content_read = read_buffer.read()
127 test_case.assertEqual(content_read, content)
129 # Seek off the end of the file and should read empty back.
130 # We run this check twice since in some cases the handle will
131 # cache knowledge of the file size.
132 read_buffer.seek(1024)
133 test_case.assertEqual(read_buffer.tell(), 1024)
134 content_read = read_buffer.read()
135 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
137 # Write two copies of the content, overwriting the single copy there.
138 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
139 write_buffer.write(double_content)
140 # Read again, this time use mode='r+', which reads what is there and
141 # then lets us write more; we'll use that to reset the file to one
142 # copy of the content.
143 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
144 test_case.assertEqual(rw_buffer.read(), double_content)
145 rw_buffer.seek(0)
146 rw_buffer.truncate()
147 rw_buffer.write(content)
148 rw_buffer.seek(0)
149 test_case.assertEqual(rw_buffer.read(), content)
150 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
151 test_case.assertEqual(read_buffer.read(), content)
152 # Append some more content to the file; should now have two copies.
153 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
154 append_buffer.write(content)
155 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
156 test_case.assertEqual(read_buffer.read(), double_content)
157 # Final mode to check is w+, which does read/write but truncates first.
158 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
159 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
160 rw_buffer.write(content)
161 rw_buffer.seek(0)
162 test_case.assertEqual(rw_buffer.read(), content)
163 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
164 test_case.assertEqual(read_buffer.read(), content)
165 # Remove file to make room for the next loop of tests with this URI.
166 uri.remove()
169if TYPE_CHECKING:
171 class TestCaseMixin(unittest.TestCase):
172 """Base class for mixin test classes that use TestCase methods."""
174 pass
176else:
178 class TestCaseMixin:
179 """Do-nothing definition of mixin base class for regular execution."""
181 pass
184class _GenericTestCase(TestCaseMixin):
185 """Generic base class for test mixin."""
187 scheme: str | None = None
188 netloc: str | None = None
189 base_path: str | None = None
190 path1 = "test_dir"
191 path2 = "file.txt"
193 def _make_uri(self, path: str, netloc: str | None = None) -> str:
194 if self.scheme is not None:
195 if netloc is None:
196 netloc = self.netloc
197 if path.startswith("/"):
198 path = path[1:]
199 if self.base_path is not None:
200 path = f"{self.base_path}/{path}".lstrip("/")
202 return f"{self.scheme}://{netloc}/{path}"
203 else:
204 return path
207class GenericTestCase(_GenericTestCase):
208 """Test cases for generic manipulation of a `ResourcePath`."""
210 def setUp(self) -> None:
211 if self.scheme is None:
212 raise unittest.SkipTest("No scheme defined")
213 self.root = self._make_uri("")
214 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
216 def test_creation(self) -> None:
217 self.assertEqual(self.root_uri.scheme, self.scheme)
218 self.assertEqual(self.root_uri.netloc, self.netloc)
219 self.assertFalse(self.root_uri.query)
220 self.assertFalse(self.root_uri.params)
222 with self.assertRaises(ValueError):
223 ResourcePath({}) # type: ignore
225 with self.assertRaises(RuntimeError):
226 ResourcePath(self.root_uri, isTemporary=True)
228 file = self.root_uri.join("file.txt")
229 with self.assertRaises(RuntimeError):
230 ResourcePath(file, forceDirectory=True)
232 with self.assertRaises(NotImplementedError):
233 ResourcePath("unknown://netloc")
235 replaced = file.replace(fragment="frag")
236 self.assertEqual(replaced.fragment, "frag")
238 with self.assertRaises(ValueError):
239 file.replace(scheme="new")
241 self.assertNotEqual(replaced, str(replaced))
242 self.assertNotEqual(str(replaced), replaced)
244 def test_extension(self) -> None:
245 uri = ResourcePath(self._make_uri("dir/test.txt"))
246 self.assertEqual(uri.updatedExtension(None), uri)
247 self.assertEqual(uri.updatedExtension(".txt"), uri)
248 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
250 fits = uri.updatedExtension(".fits.gz")
251 self.assertEqual(fits.basename(), "test.fits.gz")
252 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
254 extensionless = self.root_uri.join("no_ext")
255 self.assertEqual(extensionless.getExtension(), "")
256 extension = extensionless.updatedExtension(".fits")
257 self.assertEqual(extension.getExtension(), ".fits")
259 def test_relative(self) -> None:
260 """Check that we can get subpaths back from two URIs."""
261 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
262 self.assertTrue(parent.isdir())
263 child = parent.join("dir1/file.txt")
265 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
267 not_child = ResourcePath("/a/b/dir1/file.txt")
268 self.assertIsNone(not_child.relative_to(parent))
269 self.assertFalse(not_child.isdir())
271 not_directory = parent.join("dir1/file2.txt")
272 self.assertIsNone(child.relative_to(not_directory))
274 # Relative URIs
275 parent = ResourcePath("a/b/", forceAbsolute=False)
276 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
277 self.assertFalse(child.scheme)
278 self.assertEqual(child.relative_to(parent), "c/d.txt")
280 # forceAbsolute=True should work even on an existing ResourcePath
281 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
283 # Absolute URI and schemeless URI
284 parent = self.root_uri.join("/a/b/c/")
285 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
287 # If the child is relative and the parent is absolute we assume
288 # that the child is a child of the parent unless it uses ".."
289 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
291 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
292 self.assertIsNone(child.relative_to(parent))
294 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
295 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
297 # Test with different netloc
298 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
299 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True)
300 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
302 # This is an absolute path so will *always* return a file URI and
303 # ignore the root parameter.
304 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True)
305 self.assertEqual(parent.geturl(), "file:///a/b/c/")
307 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True)
308 child = ResourcePath("d/e.txt", root=parent)
309 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
311 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/")))
312 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
314 # Absolute schemeless child with relative parent will always fail.
315 child = ResourcePath("d/e.txt", root="/a/b/c")
316 parent = ResourcePath("d/e.txt", forceAbsolute=False)
317 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
319 def test_parents(self) -> None:
320 """Test of splitting and parent walking."""
321 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
322 child_file = parent.join("subdir/file.txt")
323 self.assertFalse(child_file.isdir())
324 child_subdir, file = child_file.split()
325 self.assertEqual(file, "file.txt")
326 self.assertTrue(child_subdir.isdir())
327 self.assertEqual(child_file.dirname(), child_subdir)
328 self.assertEqual(child_file.basename(), file)
329 self.assertEqual(child_file.parent(), child_subdir)
330 derived_parent = child_subdir.parent()
331 self.assertEqual(derived_parent, parent)
332 self.assertTrue(derived_parent.isdir())
333 self.assertEqual(child_file.parent().parent(), parent)
335 def test_escapes(self) -> None:
336 """Special characters in file paths."""
337 src = self.root_uri.join("bbb/???/test.txt")
338 self.assertNotIn("???", src.path)
339 self.assertIn("???", src.unquoted_path)
341 file = src.updatedFile("tests??.txt")
342 self.assertNotIn("??.txt", file.path)
344 src = src.updatedFile("tests??.txt")
345 self.assertIn("??.txt", src.unquoted_path)
347 # File URI and schemeless URI
348 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
349 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
350 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
352 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
353 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
355 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
356 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
358 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
360 # dir.join() morphs into a file scheme
361 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
362 new = dir.join("test_j.txt")
363 self.assertIn("???", new.unquoted_path, f"Checking {new}")
365 new2name = "###/test??.txt"
366 new2 = dir.join(new2name)
367 self.assertIn("???", new2.unquoted_path)
368 self.assertTrue(new2.unquoted_path.endswith(new2name))
370 fdir = dir.abspath()
371 self.assertNotIn("???", fdir.path)
372 self.assertIn("???", fdir.unquoted_path)
373 self.assertEqual(fdir.scheme, self.scheme)
375 fnew2 = fdir.join(new2name)
376 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
377 self.assertNotIn("###", fnew2.path)
379 # Test that children relative to schemeless and file schemes
380 # still return the same unquoted name
381 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
382 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
383 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
384 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
386 # Check for double quoting
387 plus_path = "/a/b/c+d/"
388 with self.assertLogs(level="WARNING"):
389 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
390 self.assertEqual(uri.ospath, plus_path)
392 # Check that # is not escaped for schemeless URIs
393 hash_path = "/a/b#/c&d#xyz"
394 hpos = hash_path.rfind("#")
395 uri = ResourcePath(hash_path)
396 self.assertEqual(uri.ospath, hash_path[:hpos])
397 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
399 def test_hash(self) -> None:
400 """Test that we can store URIs in sets and as keys."""
401 uri1 = self.root_uri
402 uri2 = uri1.join("test/")
403 s = {uri1, uri2}
404 self.assertIn(uri1, s)
406 d = {uri1: "1", uri2: "2"}
407 self.assertEqual(d[uri2], "2")
409 def test_root_uri(self) -> None:
410 """Test ResourcePath.root_uri()."""
411 uri = ResourcePath(self._make_uri("a/b/c.txt"))
412 self.assertEqual(uri.root_uri().geturl(), self.root)
414 def test_join(self) -> None:
415 """Test .join method."""
416 root_str = self.root
417 root = self.root_uri
419 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
420 add_dir = root.join("b/c/d/")
421 self.assertTrue(add_dir.isdir())
422 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
424 up_relative = root.join("../b/c.txt")
425 self.assertFalse(up_relative.isdir())
426 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
428 quote_example = "hsc/payload/b&c.t@x#t"
429 needs_quote = root.join(quote_example)
430 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
432 other = ResourcePath(f"{self.root}test.txt")
433 self.assertEqual(root.join(other), other)
434 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
436 joined = ResourcePath(f"{self.root}hsc/payload/").join(
437 ResourcePath("test.qgraph", forceAbsolute=False)
438 )
439 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
441 qgraph = ResourcePath("test.qgraph") # Absolute URI
442 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph)
443 self.assertEqual(joined, qgraph)
445 def test_quoting(self) -> None:
446 """Check that quoting works."""
447 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
448 subpath = "rootdir/dir1+/file?.txt"
449 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
451 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
452 self.assertEqual(child.basename(), "file?.txt")
453 self.assertEqual(child.relativeToPathRoot, subpath)
454 self.assertIn("%", child.path)
455 self.assertEqual(child.unquoted_path, "/" + subpath)
457 def test_ordering(self) -> None:
458 """Check that greater/less comparison operators work."""
459 a = self._make_uri("a.txt")
460 b = self._make_uri("b/")
461 self.assertLess(a, b)
462 self.assertFalse(a < a)
463 self.assertLessEqual(a, b)
464 self.assertLessEqual(a, a)
465 self.assertGreater(b, a)
466 self.assertFalse(b > b)
467 self.assertGreaterEqual(b, a)
468 self.assertGreaterEqual(b, b)
471class GenericReadWriteTestCase(_GenericTestCase):
472 """Test schemes that can read and write using concrete resources."""
474 transfer_modes: tuple[str, ...] = ("copy", "move")
475 testdir: str | None = None
477 def setUp(self) -> None:
478 if self.scheme is None:
479 raise unittest.SkipTest("No scheme defined")
480 self.root = self._make_uri("")
481 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
483 if self.scheme == "file":
484 # Use a local tempdir because on macOS the temp dirs use symlinks
485 # so relsymlink gets quite confused.
486 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
487 else:
488 # Create random tmp directory relative to the test root.
489 self.tmpdir = self.root_uri.join(
490 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
491 forceDirectory=True,
492 )
493 self.tmpdir.mkdir()
495 def tearDown(self) -> None:
496 if self.tmpdir and self.tmpdir.isLocal:
497 removeTestTempDir(self.tmpdir.ospath)
499 def test_file(self) -> None:
500 uri = self.tmpdir.join("test.txt")
501 self.assertFalse(uri.exists(), f"{uri} should not exist")
502 self.assertTrue(uri.path.endswith("test.txt"))
504 content = "abcdefghijklmnopqrstuv\n"
505 uri.write(content.encode())
506 self.assertTrue(uri.exists(), f"{uri} should now exist")
507 self.assertEqual(uri.read().decode(), content)
508 self.assertEqual(uri.size(), len(content.encode()))
510 with self.assertRaises(FileExistsError):
511 uri.write(b"", overwrite=False)
513 # Not all backends can tell if a remove fails so we can not
514 # test that a remove of a non-existent entry is guaranteed to raise.
515 uri.remove()
516 self.assertFalse(uri.exists())
518 # Ideally the test would remove the file again and raise a
519 # FileNotFoundError. This is not reliable for remote resources
520 # and doing an explicit check before trying to remove the resource
521 # just to raise an exception is deemed an unacceptable overhead.
523 with self.assertRaises(FileNotFoundError):
524 uri.read()
526 with self.assertRaises(FileNotFoundError):
527 self.tmpdir.join("file/not/there.txt").size()
529 # Check that creating a URI from a URI returns the same thing
530 uri2 = ResourcePath(uri)
531 self.assertEqual(uri, uri2)
532 self.assertEqual(id(uri), id(uri2))
534 def test_mkdir(self) -> None:
535 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
536 newdir.mkdir()
537 self.assertTrue(newdir.exists())
538 self.assertEqual(newdir.size(), 0)
540 newfile = newdir.join("temp.txt")
541 newfile.write(b"Data")
542 self.assertTrue(newfile.exists())
544 file = self.tmpdir.join("file.txt")
545 # Some schemes will realize that the URI is not a file and so
546 # will raise NotADirectoryError. The file scheme is more permissive
547 # and lets you write anything but will raise NotADirectoryError
548 # if a non-directory is already there. We therefore write something
549 # to the file to ensure that we trigger a portable exception.
550 file.write(b"")
551 with self.assertRaises(NotADirectoryError):
552 file.mkdir()
554 # The root should exist.
555 self.root_uri.mkdir()
556 self.assertTrue(self.root_uri.exists())
558 def test_transfer(self) -> None:
559 src = self.tmpdir.join("test.txt")
560 content = "Content is some content\nwith something to say\n\n"
561 src.write(content.encode())
563 can_move = "move" in self.transfer_modes
564 for mode in self.transfer_modes:
565 if mode == "move":
566 continue
568 dest = self.tmpdir.join(f"dest_{mode}.txt")
569 # Ensure that we get some debugging output.
570 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
571 dest.transfer_from(src, transfer=mode)
572 self.assertIn("Transferring ", "\n".join(cm.output))
573 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
575 new_content = dest.read().decode()
576 self.assertEqual(new_content, content)
578 if mode in ("symlink", "relsymlink"):
579 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
581 # If the source and destination are hardlinks of each other
582 # the transfer should work even if overwrite=False.
583 if mode in ("link", "hardlink"):
584 dest.transfer_from(src, transfer=mode)
585 else:
586 with self.assertRaises(
587 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
588 ):
589 dest.transfer_from(src, transfer=mode)
591 # Transfer again and overwrite.
592 dest.transfer_from(src, transfer=mode, overwrite=True)
594 dest.remove()
596 b = src.read()
597 self.assertEqual(b.decode(), new_content)
599 nbytes = 10
600 subset = src.read(size=nbytes)
601 self.assertEqual(len(subset), nbytes)
602 self.assertEqual(subset.decode(), content[:nbytes])
604 # Transferring to self should be okay.
605 src.transfer_from(src, "auto")
607 with self.assertRaises(ValueError):
608 src.transfer_from(src, transfer="unknown")
610 # A move transfer is special.
611 if can_move:
612 dest.transfer_from(src, transfer="move")
613 self.assertFalse(src.exists())
614 self.assertTrue(dest.exists())
615 else:
616 src.remove()
618 dest.remove()
619 with self.assertRaises(FileNotFoundError):
620 dest.transfer_from(src, "auto")
622 def test_local_transfer(self) -> None:
623 """Test we can transfer to and from local file."""
624 remote_src = self.tmpdir.join("src.json")
625 remote_src.write(b"42")
626 remote_dest = self.tmpdir.join("dest.json")
628 with ResourcePath.temporary_uri(suffix=".json") as tmp:
629 self.assertTrue(tmp.isLocal)
630 tmp.transfer_from(remote_src, transfer="auto")
631 self.assertEqual(tmp.read(), remote_src.read())
633 remote_dest.transfer_from(tmp, transfer="auto")
634 self.assertEqual(remote_dest.read(), tmp.read())
636 # Temporary (possibly remote) resource.
637 # Transfers between temporary resources.
638 with (
639 ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp,
640 ResourcePath.temporary_uri(suffix=".json") as local_tmp,
641 ):
642 remote_tmp.write(b"42")
643 if not remote_tmp.isLocal:
644 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
645 with self.assertRaises(RuntimeError):
646 # Trying to symlink a remote resource is not going
647 # to work. A hardlink could work but would rely
648 # on the local temp space being on the same
649 # filesystem as the target.
650 local_tmp.transfer_from(remote_tmp, transfer)
651 local_tmp.transfer_from(remote_tmp, "move")
652 self.assertFalse(remote_tmp.exists())
653 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
654 self.assertEqual(local_tmp.read(), remote_tmp.read())
656 # Transfer of missing remote.
657 remote_tmp.remove()
658 with self.assertRaises(FileNotFoundError):
659 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
661 def test_local(self) -> None:
662 """Check that remote resources can be made local."""
663 src = self.tmpdir.join("test.txt")
664 original_content = "Content is some content\nwith something to say\n\n"
665 src.write(original_content.encode())
667 # Run this twice to ensure use of cache in code coverage
668 # if applicable.
669 for _ in (1, 2):
670 with src.as_local() as local_uri:
671 self.assertTrue(local_uri.isLocal)
672 content = local_uri.read().decode()
673 self.assertEqual(content, original_content)
675 if src.isLocal:
676 self.assertEqual(src, local_uri)
678 with self.assertRaises(IsADirectoryError):
679 with self.root_uri.as_local() as local_uri:
680 pass
682 def test_walk(self) -> None:
683 """Walk a directory hierarchy."""
684 root = self.tmpdir.join("walk/")
686 # Look for a file that is not there
687 file = root.join("config/basic/butler.yaml")
688 found_list = list(ResourcePath.findFileResources([file]))
689 self.assertEqual(found_list[0], file)
691 # First create the files (content is irrelevant).
692 expected_files = {
693 "dir1/a.yaml",
694 "dir1/b.yaml",
695 "dir1/c.json",
696 "dir2/d.json",
697 "dir2/e.yaml",
698 }
699 expected_uris = {root.join(f) for f in expected_files}
700 for uri in expected_uris:
701 uri.write(b"")
702 self.assertTrue(uri.exists())
704 # Look for the files.
705 found = set(ResourcePath.findFileResources([root]))
706 self.assertEqual(found, expected_uris)
708 # Now solely the YAML files.
709 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
710 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
711 self.assertEqual(found, expected_yaml)
713 # Now two explicit directories and a file
714 expected = set(expected_yaml)
715 expected.add(file)
717 found = set(
718 ResourcePath.findFileResources(
719 [file, root.join("dir1/"), root.join("dir2/")],
720 file_filter=r".*\.yaml$",
721 )
722 )
723 self.assertEqual(found, expected)
725 # Group by directory -- find everything and compare it with what
726 # we expected to be there in total.
727 found_yaml = set()
728 counter = 0
729 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
730 assert not isinstance(uris, ResourcePath) # for mypy.
731 found_uris = set(uris)
732 if found_uris:
733 counter += 1
735 found_yaml.update(found_uris)
737 expected_yaml_2 = expected_yaml
738 expected_yaml_2.add(file)
739 self.assertEqual(found_yaml, expected_yaml)
740 self.assertEqual(counter, 3)
742 # Grouping but check that single files are returned in a single group
743 # at the end
744 file2 = root.join("config/templates/templates-bad.yaml")
745 found_grouped = [
746 list(group)
747 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
748 if not isinstance(group, ResourcePath) # For mypy.
749 ]
750 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
751 self.assertEqual(list(found_grouped[1]), [file, file2])
753 with self.assertRaises(ValueError):
754 # The list forces the generator to run.
755 list(file.walk())
757 # A directory that does not exist returns nothing.
758 self.assertEqual(list(root.join("dir3/").walk()), [])
760 def test_large_walk(self) -> None:
761 # In some systems pagination is used so ensure that we can handle
762 # large numbers of files. For example S3 limits us to 1000 responses
763 # per listing call.
764 created = set()
765 counter = 1
766 n_dir1 = 1100
767 root = self.tmpdir.join("large_walk", forceDirectory=True)
768 while counter <= n_dir1:
769 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
770 new.write(f"{counter}".encode())
771 created.add(new)
772 counter += 1
773 counter = 1
774 # Put some in a subdirectory to make sure we are looking in a
775 # hierarchy.
776 n_dir2 = 100
777 subdir = root.join("subdir", forceDirectory=True)
778 while counter <= n_dir2:
779 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
780 new.write(f"{counter}".encode())
781 created.add(new)
782 counter += 1
784 found = set(ResourcePath.findFileResources([root]))
785 self.assertEqual(len(found), n_dir1 + n_dir2)
786 self.assertEqual(found, created)
788 # Again with grouping.
789 # (mypy gets upset not knowing which of the two options is being
790 # returned so add useless instance check).
791 found_list = [
792 list(group)
793 for group in ResourcePath.findFileResources([root], grouped=True)
794 if not isinstance(group, ResourcePath) # For mypy.
795 ]
796 self.assertEqual(len(found_list), 2)
797 self.assertEqual(len(found_list[0]), n_dir1)
798 self.assertEqual(len(found_list[1]), n_dir2)
800 def test_temporary(self) -> None:
801 prefix = self.tmpdir.join("tmp", forceDirectory=True)
802 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
803 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
804 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
805 self.assertFalse(tmp.exists(), f"uri: {tmp}")
806 tmp.write(b"abcd")
807 self.assertTrue(tmp.exists(), f"uri: {tmp}")
808 self.assertTrue(tmp.isTemporary)
809 self.assertFalse(tmp.exists(), f"uri: {tmp}")
811 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
812 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
813 # Use a specified tmpdir and check it is okay for the file
814 # to not be created.
815 self.assertFalse(tmp.getExtension())
816 self.assertFalse(tmp.exists(), f"uri: {tmp}")
817 self.assertEqual(tmp.scheme, self.scheme)
818 self.assertTrue(tmp.isTemporary)
819 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
821 # Fake a directory suffix.
822 with self.assertRaises(NotImplementedError):
823 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
824 pass
826 def test_open(self) -> None:
827 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
828 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
829 _check_open(self, tmp, mode_suffixes=("", "t"))
830 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
831 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
832 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
833 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
834 _check_open(self, tmp, mode_suffixes=("b",))
835 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
837 with self.assertRaises(IsADirectoryError):
838 with self.root_uri.open():
839 pass
841 def test_mexists(self) -> None:
842 root = self.tmpdir.join("mexists/")
844 # A file that is not there.
845 file = root.join("config/basic/butler.yaml")
847 # Create some files.
848 expected_files = {
849 "dir1/a.yaml",
850 "dir1/b.yaml",
851 "dir2/e.yaml",
852 }
853 expected_uris = {root.join(f) for f in expected_files}
854 for uri in expected_uris:
855 uri.write(b"")
856 self.assertTrue(uri.exists())
857 expected_uris.add(file)
859 multi = ResourcePath.mexists(expected_uris)
861 for uri, is_there in multi.items():
862 if uri == file:
863 self.assertFalse(is_there)
864 else:
865 self.assertTrue(is_there)