Coverage for python/lsst/resources/tests.py: 9%
528 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-25 09:29 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-25 09:29 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from collections.abc import Callable, Iterable
24from typing import Any
26from lsst.resources import ResourcePath
27from lsst.resources.utils import makeTestTempDir, removeTestTempDir
29TESTDIR = os.path.abspath(os.path.dirname(__file__))
32def _check_open(
33 test_case: _GenericTestCase | unittest.TestCase,
34 uri: ResourcePath,
35 *,
36 mode_suffixes: Iterable[str] = ("", "t", "b"),
37 **kwargs: Any,
38) -> None:
39 """Test an implementation of ButlerURI.open.
41 Parameters
42 ----------
43 test_case : `unittest.TestCase`
44 Test case to use for assertions.
45 uri : `ResourcePath`
46 URI to use for tests. Must point to a writeable location that is not
47 yet occupied by a file. On return, the location may point to a file
48 only if the test fails.
49 mode_suffixes : `~collections.abc.Iterable` of `str`
50 Suffixes to pass as part of the ``mode`` argument to
51 `ResourcePath.open`, indicating whether to open as binary or as text;
52 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
53 **kwargs
54 Additional keyword arguments to forward to all calls to `open`.
55 """
56 text_content = "abcdefghijklmnopqrstuvwxyz🙂"
57 bytes_content = uuid.uuid4().bytes
58 content_by_mode_suffix = {
59 "": text_content,
60 "t": text_content,
61 "b": bytes_content,
62 }
63 empty_content_by_mode_suffix = {
64 "": "",
65 "t": "",
66 "b": b"",
67 }
68 # To appease mypy
69 double_content_by_mode_suffix = {
70 "": text_content + text_content,
71 "t": text_content + text_content,
72 "b": bytes_content + bytes_content,
73 }
74 for mode_suffix in mode_suffixes:
75 content = content_by_mode_suffix[mode_suffix]
76 double_content = double_content_by_mode_suffix[mode_suffix]
77 # Create file with mode='x', which prohibits overwriting.
78 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
79 write_buffer.write(content)
80 test_case.assertTrue(uri.exists())
81 # Check that opening with 'x' now raises, and does not modify content.
82 with test_case.assertRaises(FileExistsError):
83 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
84 write_buffer.write("bad")
85 # Read the file we created and check the contents.
86 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
87 test_case.assertEqual(read_buffer.read(), content)
88 # Check that we can read bytes in a loop and get EOF
89 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
90 # Seek off the end of the file and should read empty back.
91 read_buffer.seek(1024)
92 test_case.assertEqual(read_buffer.tell(), 1024)
93 content_read = read_buffer.read() # Read as much as we can.
94 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
96 # First read more than the content.
97 read_buffer.seek(0)
98 size = len(content) * 3
99 chunk_read = read_buffer.read(size)
100 test_case.assertEqual(chunk_read, content)
102 # Repeated reads should always return empty string.
103 chunk_read = read_buffer.read(size)
104 test_case.assertEqual(len(chunk_read), 0)
105 chunk_read = read_buffer.read(size)
106 test_case.assertEqual(len(chunk_read), 0)
108 # Go back to start of file and read in smaller chunks.
109 read_buffer.seek(0)
110 size = len(content) // 3
112 content_read = empty_content_by_mode_suffix[mode_suffix]
113 n_reads = 0
114 while chunk_read := read_buffer.read(size):
115 content_read += chunk_read
116 n_reads += 1
117 if n_reads > 10: # In case EOF never hits because of bug.
118 raise AssertionError(
119 f"Failed to stop reading from file after {n_reads} loops. "
120 f"Read {len(content_read)} bytes/characters. Expected {len(content)}."
121 )
122 test_case.assertEqual(content_read, content)
124 # Go back to start of file and read the entire thing.
125 read_buffer.seek(0)
126 content_read = read_buffer.read()
127 test_case.assertEqual(content_read, content)
129 # Seek off the end of the file and should read empty back.
130 # We run this check twice since in some cases the handle will
131 # cache knowledge of the file size.
132 read_buffer.seek(1024)
133 test_case.assertEqual(read_buffer.tell(), 1024)
134 content_read = read_buffer.read()
135 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
137 # Write two copies of the content, overwriting the single copy there.
138 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
139 write_buffer.write(double_content)
140 # Read again, this time use mode='r+', which reads what is there and
141 # then lets us write more; we'll use that to reset the file to one
142 # copy of the content.
143 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
144 test_case.assertEqual(rw_buffer.read(), double_content)
145 rw_buffer.seek(0)
146 rw_buffer.truncate()
147 rw_buffer.write(content)
148 rw_buffer.seek(0)
149 test_case.assertEqual(rw_buffer.read(), content)
150 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
151 test_case.assertEqual(read_buffer.read(), content)
152 # Append some more content to the file; should now have two copies.
153 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
154 append_buffer.write(content)
155 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
156 test_case.assertEqual(read_buffer.read(), double_content)
157 # Final mode to check is w+, which does read/write but truncates first.
158 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
159 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
160 rw_buffer.write(content)
161 rw_buffer.seek(0)
162 test_case.assertEqual(rw_buffer.read(), content)
163 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
164 test_case.assertEqual(read_buffer.read(), content)
165 # Remove file to make room for the next loop of tests with this URI.
166 uri.remove()
169class _GenericTestCase:
170 """Generic base class for test mixin."""
172 scheme: str | None = None
173 netloc: str | None = None
174 base_path: str | None = None
175 path1 = "test_dir"
176 path2 = "file.txt"
178 # Because we use a mixin for tests mypy needs to understand that
179 # the unittest.TestCase methods exist.
180 # We do not inherit from unittest.TestCase because that results
181 # in the tests defined here being run as well as the tests in the
182 # test file itself. We can make those tests skip but it gives an
183 # uniformative view of how many tests are running.
184 assertEqual: Callable
185 assertNotEqual: Callable
186 assertIsNone: Callable
187 assertIn: Callable
188 assertNotIn: Callable
189 assertFalse: Callable
190 assertTrue: Callable
191 assertRaises: Callable
192 assertLogs: Callable
194 def _make_uri(self, path: str, netloc: str | None = None) -> str:
195 if self.scheme is not None:
196 if netloc is None:
197 netloc = self.netloc
198 if path.startswith("/"):
199 path = path[1:]
200 if self.base_path is not None:
201 path = f"{self.base_path}/{path}".lstrip("/")
203 return f"{self.scheme}://{netloc}/{path}"
204 else:
205 return path
208class GenericTestCase(_GenericTestCase):
209 """Test cases for generic manipulation of a `ResourcePath`"""
211 def setUp(self) -> None:
212 if self.scheme is None:
213 raise unittest.SkipTest("No scheme defined")
214 self.root = self._make_uri("")
215 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
217 def test_creation(self) -> None:
218 self.assertEqual(self.root_uri.scheme, self.scheme)
219 self.assertEqual(self.root_uri.netloc, self.netloc)
220 self.assertFalse(self.root_uri.query)
221 self.assertFalse(self.root_uri.params)
223 with self.assertRaises(ValueError):
224 ResourcePath({}) # type: ignore
226 with self.assertRaises(RuntimeError):
227 ResourcePath(self.root_uri, isTemporary=True)
229 file = self.root_uri.join("file.txt")
230 with self.assertRaises(RuntimeError):
231 ResourcePath(file, forceDirectory=True)
233 with self.assertRaises(NotImplementedError):
234 ResourcePath("unknown://netloc")
236 replaced = file.replace(fragment="frag")
237 self.assertEqual(replaced.fragment, "frag")
239 with self.assertRaises(ValueError):
240 file.replace(scheme="new")
242 self.assertNotEqual(replaced, str(replaced))
243 self.assertNotEqual(str(replaced), replaced)
245 def test_extension(self) -> None:
246 uri = ResourcePath(self._make_uri("dir/test.txt"))
247 self.assertEqual(uri.updatedExtension(None), uri)
248 self.assertEqual(uri.updatedExtension(".txt"), uri)
249 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
251 fits = uri.updatedExtension(".fits.gz")
252 self.assertEqual(fits.basename(), "test.fits.gz")
253 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
255 extensionless = self.root_uri.join("no_ext")
256 self.assertEqual(extensionless.getExtension(), "")
257 extension = extensionless.updatedExtension(".fits")
258 self.assertEqual(extension.getExtension(), ".fits")
260 def test_relative(self) -> None:
261 """Check that we can get subpaths back from two URIs"""
262 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
263 self.assertTrue(parent.isdir())
264 child = parent.join("dir1/file.txt")
266 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
268 not_child = ResourcePath("/a/b/dir1/file.txt")
269 self.assertIsNone(not_child.relative_to(parent))
270 self.assertFalse(not_child.isdir())
272 not_directory = parent.join("dir1/file2.txt")
273 self.assertIsNone(child.relative_to(not_directory))
275 # Relative URIs
276 parent = ResourcePath("a/b/", forceAbsolute=False)
277 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
278 self.assertFalse(child.scheme)
279 self.assertEqual(child.relative_to(parent), "c/d.txt")
281 # forceAbsolute=True should work even on an existing ResourcePath
282 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
284 # Absolute URI and schemeless URI
285 parent = self.root_uri.join("/a/b/c/")
286 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
288 # If the child is relative and the parent is absolute we assume
289 # that the child is a child of the parent unless it uses ".."
290 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
292 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
293 self.assertIsNone(child.relative_to(parent))
295 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
296 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
298 # Test with different netloc
299 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
300 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True)
301 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
303 # This is an absolute path so will *always* return a file URI and
304 # ignore the root parameter.
305 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True)
306 self.assertEqual(parent.geturl(), "file:///a/b/c/")
308 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True)
309 child = ResourcePath("d/e.txt", root=parent)
310 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
312 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/")))
313 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
315 # Absolute schemeless child with relative parent will always fail.
316 child = ResourcePath("d/e.txt", root="/a/b/c")
317 parent = ResourcePath("d/e.txt", forceAbsolute=False)
318 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
320 def test_parents(self) -> None:
321 """Test of splitting and parent walking."""
322 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
323 child_file = parent.join("subdir/file.txt")
324 self.assertFalse(child_file.isdir())
325 child_subdir, file = child_file.split()
326 self.assertEqual(file, "file.txt")
327 self.assertTrue(child_subdir.isdir())
328 self.assertEqual(child_file.dirname(), child_subdir)
329 self.assertEqual(child_file.basename(), file)
330 self.assertEqual(child_file.parent(), child_subdir)
331 derived_parent = child_subdir.parent()
332 self.assertEqual(derived_parent, parent)
333 self.assertTrue(derived_parent.isdir())
334 self.assertEqual(child_file.parent().parent(), parent)
336 def test_escapes(self) -> None:
337 """Special characters in file paths"""
338 src = self.root_uri.join("bbb/???/test.txt")
339 self.assertNotIn("???", src.path)
340 self.assertIn("???", src.unquoted_path)
342 file = src.updatedFile("tests??.txt")
343 self.assertNotIn("??.txt", file.path)
345 src = src.updatedFile("tests??.txt")
346 self.assertIn("??.txt", src.unquoted_path)
348 # File URI and schemeless URI
349 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
350 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
351 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
353 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
354 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
356 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
357 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
359 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
361 # dir.join() morphs into a file scheme
362 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
363 new = dir.join("test_j.txt")
364 self.assertIn("???", new.unquoted_path, f"Checking {new}")
366 new2name = "###/test??.txt"
367 new2 = dir.join(new2name)
368 self.assertIn("???", new2.unquoted_path)
369 self.assertTrue(new2.unquoted_path.endswith(new2name))
371 fdir = dir.abspath()
372 self.assertNotIn("???", fdir.path)
373 self.assertIn("???", fdir.unquoted_path)
374 self.assertEqual(fdir.scheme, self.scheme)
376 fnew2 = fdir.join(new2name)
377 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
378 self.assertNotIn("###", fnew2.path)
380 # Test that children relative to schemeless and file schemes
381 # still return the same unquoted name
382 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
383 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
384 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
385 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
387 # Check for double quoting
388 plus_path = "/a/b/c+d/"
389 with self.assertLogs(level="WARNING"):
390 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
391 self.assertEqual(uri.ospath, plus_path)
393 # Check that # is not escaped for schemeless URIs
394 hash_path = "/a/b#/c&d#xyz"
395 hpos = hash_path.rfind("#")
396 uri = ResourcePath(hash_path)
397 self.assertEqual(uri.ospath, hash_path[:hpos])
398 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
400 def test_hash(self) -> None:
401 """Test that we can store URIs in sets and as keys."""
402 uri1 = self.root_uri
403 uri2 = uri1.join("test/")
404 s = {uri1, uri2}
405 self.assertIn(uri1, s)
407 d = {uri1: "1", uri2: "2"}
408 self.assertEqual(d[uri2], "2")
410 def test_root_uri(self) -> None:
411 """Test ResourcePath.root_uri()."""
412 uri = ResourcePath(self._make_uri("a/b/c.txt"))
413 self.assertEqual(uri.root_uri().geturl(), self.root)
415 def test_join(self) -> None:
416 """Test .join method."""
417 root_str = self.root
418 root = self.root_uri
420 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
421 add_dir = root.join("b/c/d/")
422 self.assertTrue(add_dir.isdir())
423 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
425 up_relative = root.join("../b/c.txt")
426 self.assertFalse(up_relative.isdir())
427 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
429 quote_example = "hsc/payload/b&c.t@x#t"
430 needs_quote = root.join(quote_example)
431 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
433 other = ResourcePath(f"{self.root}test.txt")
434 self.assertEqual(root.join(other), other)
435 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
437 joined = ResourcePath(f"{self.root}hsc/payload/").join(
438 ResourcePath("test.qgraph", forceAbsolute=False)
439 )
440 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
442 qgraph = ResourcePath("test.qgraph") # Absolute URI
443 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph)
444 self.assertEqual(joined, qgraph)
446 def test_quoting(self) -> None:
447 """Check that quoting works."""
448 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
449 subpath = "rootdir/dir1+/file?.txt"
450 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
452 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
453 self.assertEqual(child.basename(), "file?.txt")
454 self.assertEqual(child.relativeToPathRoot, subpath)
455 self.assertIn("%", child.path)
456 self.assertEqual(child.unquoted_path, "/" + subpath)
458 def test_ordering(self) -> None:
459 """Check that greater/less comparison operators work."""
460 a = self._make_uri("a.txt")
461 b = self._make_uri("b/")
462 self.assertTrue(a < b)
463 self.assertFalse(a < a)
464 self.assertTrue(a <= b)
465 self.assertTrue(a <= a)
466 self.assertTrue(b > a)
467 self.assertFalse(b > b)
468 self.assertTrue(b >= a)
469 self.assertTrue(b >= b)
472class GenericReadWriteTestCase(_GenericTestCase):
473 """Test schemes that can read and write using concrete resources."""
475 transfer_modes: tuple[str, ...] = ("copy", "move")
476 testdir: str | None = None
478 def setUp(self) -> None:
479 if self.scheme is None:
480 raise unittest.SkipTest("No scheme defined")
481 self.root = self._make_uri("")
482 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
484 if self.scheme == "file":
485 # Use a local tempdir because on macOS the temp dirs use symlinks
486 # so relsymlink gets quite confused.
487 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
488 else:
489 # Create random tmp directory relative to the test root.
490 self.tmpdir = self.root_uri.join(
491 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
492 forceDirectory=True,
493 )
494 self.tmpdir.mkdir()
496 def tearDown(self) -> None:
497 if self.tmpdir:
498 if self.tmpdir.isLocal:
499 removeTestTempDir(self.tmpdir.ospath)
501 def test_file(self) -> None:
502 uri = self.tmpdir.join("test.txt")
503 self.assertFalse(uri.exists(), f"{uri} should not exist")
504 self.assertTrue(uri.path.endswith("test.txt"))
506 content = "abcdefghijklmnopqrstuv\n"
507 uri.write(content.encode())
508 self.assertTrue(uri.exists(), f"{uri} should now exist")
509 self.assertEqual(uri.read().decode(), content)
510 self.assertEqual(uri.size(), len(content.encode()))
512 with self.assertRaises(FileExistsError):
513 uri.write(b"", overwrite=False)
515 # Not all backends can tell if a remove fails so we can not
516 # test that a remove of a non-existent entry is guaranteed to raise.
517 uri.remove()
518 self.assertFalse(uri.exists())
520 # Ideally the test would remove the file again and raise a
521 # FileNotFoundError. This is not reliable for remote resources
522 # and doing an explicit check before trying to remove the resource
523 # just to raise an exception is deemed an unacceptable overhead.
525 with self.assertRaises(FileNotFoundError):
526 uri.read()
528 with self.assertRaises(FileNotFoundError):
529 self.tmpdir.join("file/not/there.txt").size()
531 # Check that creating a URI from a URI returns the same thing
532 uri2 = ResourcePath(uri)
533 self.assertEqual(uri, uri2)
534 self.assertEqual(id(uri), id(uri2))
536 def test_mkdir(self) -> None:
537 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
538 newdir.mkdir()
539 self.assertTrue(newdir.exists())
540 self.assertEqual(newdir.size(), 0)
542 newfile = newdir.join("temp.txt")
543 newfile.write(b"Data")
544 self.assertTrue(newfile.exists())
546 file = self.tmpdir.join("file.txt")
547 # Some schemes will realize that the URI is not a file and so
548 # will raise NotADirectoryError. The file scheme is more permissive
549 # and lets you write anything but will raise NotADirectoryError
550 # if a non-directory is already there. We therefore write something
551 # to the file to ensure that we trigger a portable exception.
552 file.write(b"")
553 with self.assertRaises(NotADirectoryError):
554 file.mkdir()
556 # The root should exist.
557 self.root_uri.mkdir()
558 self.assertTrue(self.root_uri.exists())
560 def test_transfer(self) -> None:
561 src = self.tmpdir.join("test.txt")
562 content = "Content is some content\nwith something to say\n\n"
563 src.write(content.encode())
565 can_move = "move" in self.transfer_modes
566 for mode in self.transfer_modes:
567 if mode == "move":
568 continue
570 dest = self.tmpdir.join(f"dest_{mode}.txt")
571 # Ensure that we get some debugging output.
572 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
573 dest.transfer_from(src, transfer=mode)
574 self.assertIn("Transferring ", "\n".join(cm.output))
575 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
577 new_content = dest.read().decode()
578 self.assertEqual(new_content, content)
580 if mode in ("symlink", "relsymlink"):
581 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
583 # If the source and destination are hardlinks of each other
584 # the transfer should work even if overwrite=False.
585 if mode in ("link", "hardlink"):
586 dest.transfer_from(src, transfer=mode)
587 else:
588 with self.assertRaises(
589 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
590 ):
591 dest.transfer_from(src, transfer=mode)
593 # Transfer again and overwrite.
594 dest.transfer_from(src, transfer=mode, overwrite=True)
596 dest.remove()
598 b = src.read()
599 self.assertEqual(b.decode(), new_content)
601 nbytes = 10
602 subset = src.read(size=nbytes)
603 self.assertEqual(len(subset), nbytes)
604 self.assertEqual(subset.decode(), content[:nbytes])
606 # Transferring to self should be okay.
607 src.transfer_from(src, "auto")
609 with self.assertRaises(ValueError):
610 src.transfer_from(src, transfer="unknown")
612 # A move transfer is special.
613 if can_move:
614 dest.transfer_from(src, transfer="move")
615 self.assertFalse(src.exists())
616 self.assertTrue(dest.exists())
617 else:
618 src.remove()
620 dest.remove()
621 with self.assertRaises(FileNotFoundError):
622 dest.transfer_from(src, "auto")
624 def test_local_transfer(self) -> None:
625 """Test we can transfer to and from local file."""
626 remote_src = self.tmpdir.join("src.json")
627 remote_src.write(b"42")
628 remote_dest = self.tmpdir.join("dest.json")
630 with ResourcePath.temporary_uri(suffix=".json") as tmp:
631 self.assertTrue(tmp.isLocal)
632 tmp.transfer_from(remote_src, transfer="auto")
633 self.assertEqual(tmp.read(), remote_src.read())
635 remote_dest.transfer_from(tmp, transfer="auto")
636 self.assertEqual(remote_dest.read(), tmp.read())
638 # Temporary (possibly remote) resource.
639 # Transfers between temporary resources.
640 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
641 # Temporary local resource.
642 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
643 remote_tmp.write(b"42")
644 if not remote_tmp.isLocal:
645 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
646 with self.assertRaises(RuntimeError):
647 # Trying to symlink a remote resource is not going
648 # to work. A hardlink could work but would rely
649 # on the local temp space being on the same
650 # filesystem as the target.
651 local_tmp.transfer_from(remote_tmp, transfer)
652 local_tmp.transfer_from(remote_tmp, "move")
653 self.assertFalse(remote_tmp.exists())
654 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
655 self.assertEqual(local_tmp.read(), remote_tmp.read())
657 # Transfer of missing remote.
658 remote_tmp.remove()
659 with self.assertRaises(FileNotFoundError):
660 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
662 def test_local(self) -> None:
663 """Check that remote resources can be made local."""
664 src = self.tmpdir.join("test.txt")
665 original_content = "Content is some content\nwith something to say\n\n"
666 src.write(original_content.encode())
668 # Run this twice to ensure use of cache in code coverage
669 # if applicable.
670 for _ in (1, 2):
671 with src.as_local() as local_uri:
672 self.assertTrue(local_uri.isLocal)
673 content = local_uri.read().decode()
674 self.assertEqual(content, original_content)
676 if src.isLocal:
677 self.assertEqual(src, local_uri)
679 with self.assertRaises(IsADirectoryError):
680 with self.root_uri.as_local() as local_uri:
681 pass
683 def test_walk(self) -> None:
684 """Walk a directory hierarchy."""
685 root = self.tmpdir.join("walk/")
687 # Look for a file that is not there
688 file = root.join("config/basic/butler.yaml")
689 found_list = list(ResourcePath.findFileResources([file]))
690 self.assertEqual(found_list[0], file)
692 # First create the files (content is irrelevant).
693 expected_files = {
694 "dir1/a.yaml",
695 "dir1/b.yaml",
696 "dir1/c.json",
697 "dir2/d.json",
698 "dir2/e.yaml",
699 }
700 expected_uris = {root.join(f) for f in expected_files}
701 for uri in expected_uris:
702 uri.write(b"")
703 self.assertTrue(uri.exists())
705 # Look for the files.
706 found = set(ResourcePath.findFileResources([root]))
707 self.assertEqual(found, expected_uris)
709 # Now solely the YAML files.
710 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
711 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
712 self.assertEqual(found, expected_yaml)
714 # Now two explicit directories and a file
715 expected = set(expected_yaml)
716 expected.add(file)
718 found = set(
719 ResourcePath.findFileResources(
720 [file, root.join("dir1/"), root.join("dir2/")],
721 file_filter=r".*\.yaml$",
722 )
723 )
724 self.assertEqual(found, expected)
726 # Group by directory -- find everything and compare it with what
727 # we expected to be there in total.
728 found_yaml = set()
729 counter = 0
730 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
731 assert not isinstance(uris, ResourcePath) # for mypy.
732 found_uris = set(uris)
733 if found_uris:
734 counter += 1
736 found_yaml.update(found_uris)
738 expected_yaml_2 = expected_yaml
739 expected_yaml_2.add(file)
740 self.assertEqual(found_yaml, expected_yaml)
741 self.assertEqual(counter, 3)
743 # Grouping but check that single files are returned in a single group
744 # at the end
745 file2 = root.join("config/templates/templates-bad.yaml")
746 found_grouped = [
747 list(group)
748 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
749 if not isinstance(group, ResourcePath) # For mypy.
750 ]
751 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
752 self.assertEqual(list(found_grouped[1]), [file, file2])
754 with self.assertRaises(ValueError):
755 # The list forces the generator to run.
756 list(file.walk())
758 # A directory that does not exist returns nothing.
759 self.assertEqual(list(root.join("dir3/").walk()), [])
761 def test_large_walk(self) -> None:
762 # In some systems pagination is used so ensure that we can handle
763 # large numbers of files. For example S3 limits us to 1000 responses
764 # per listing call.
765 created = set()
766 counter = 1
767 n_dir1 = 1100
768 root = self.tmpdir.join("large_walk", forceDirectory=True)
769 while counter <= n_dir1:
770 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
771 new.write(f"{counter}".encode())
772 created.add(new)
773 counter += 1
774 counter = 1
775 # Put some in a subdirectory to make sure we are looking in a
776 # hierarchy.
777 n_dir2 = 100
778 subdir = root.join("subdir", forceDirectory=True)
779 while counter <= n_dir2:
780 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
781 new.write(f"{counter}".encode())
782 created.add(new)
783 counter += 1
785 found = set(ResourcePath.findFileResources([root]))
786 self.assertEqual(len(found), n_dir1 + n_dir2)
787 self.assertEqual(found, created)
789 # Again with grouping.
790 # (mypy gets upset not knowing which of the two options is being
791 # returned so add useless instance check).
792 found_list = [
793 list(group)
794 for group in ResourcePath.findFileResources([root], grouped=True)
795 if not isinstance(group, ResourcePath) # For mypy.
796 ]
797 self.assertEqual(len(found_list), 2)
798 self.assertEqual(len(found_list[0]), n_dir1)
799 self.assertEqual(len(found_list[1]), n_dir2)
801 def test_temporary(self) -> None:
802 prefix = self.tmpdir.join("tmp", forceDirectory=True)
803 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
804 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
805 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
806 self.assertFalse(tmp.exists(), f"uri: {tmp}")
807 tmp.write(b"abcd")
808 self.assertTrue(tmp.exists(), f"uri: {tmp}")
809 self.assertTrue(tmp.isTemporary)
810 self.assertFalse(tmp.exists(), f"uri: {tmp}")
812 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
813 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
814 # Use a specified tmpdir and check it is okay for the file
815 # to not be created.
816 self.assertFalse(tmp.getExtension())
817 self.assertFalse(tmp.exists(), f"uri: {tmp}")
818 self.assertEqual(tmp.scheme, self.scheme)
819 self.assertTrue(tmp.isTemporary)
820 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
822 # Fake a directory suffix.
823 with self.assertRaises(NotImplementedError):
824 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
825 pass
827 def test_open(self) -> None:
828 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
829 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
830 _check_open(self, tmp, mode_suffixes=("", "t"))
831 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
832 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
833 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
834 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
835 _check_open(self, tmp, mode_suffixes=("b",))
836 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
838 with self.assertRaises(IsADirectoryError):
839 with self.root_uri.open():
840 pass
842 def test_mexists(self) -> None:
843 root = self.tmpdir.join("mexists/")
845 # A file that is not there.
846 file = root.join("config/basic/butler.yaml")
848 # Create some files.
849 expected_files = {
850 "dir1/a.yaml",
851 "dir1/b.yaml",
852 "dir2/e.yaml",
853 }
854 expected_uris = {root.join(f) for f in expected_files}
855 for uri in expected_uris:
856 uri.write(b"")
857 self.assertTrue(uri.exists())
858 expected_uris.add(file)
860 multi = ResourcePath.mexists(expected_uris)
862 for uri, is_there in multi.items():
863 if uri == file:
864 self.assertFalse(is_there)
865 else:
866 self.assertTrue(is_there)