Coverage for python/lsst/resources/tests.py: 9%
527 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:04 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-12 02:04 -0700
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from typing import Any, Callable, Iterable, Optional, Union
25from lsst.resources import ResourcePath
26from lsst.resources.utils import makeTestTempDir, removeTestTempDir
28TESTDIR = os.path.abspath(os.path.dirname(__file__))
31def _check_open(
32 test_case: Union[_GenericTestCase, unittest.TestCase],
33 uri: ResourcePath,
34 *,
35 mode_suffixes: Iterable[str] = ("", "t", "b"),
36 **kwargs: Any,
37) -> None:
38 """Test an implementation of ButlerURI.open.
40 Parameters
41 ----------
42 test_case : `unittest.TestCase`
43 Test case to use for assertions.
44 uri : `ResourcePath`
45 URI to use for tests. Must point to a writeable location that is not
46 yet occupied by a file. On return, the location may point to a file
47 only if the test fails.
48 mode_suffixes : `Iterable` of `str`
49 Suffixes to pass as part of the ``mode`` argument to
50 `ResourcePath.open`, indicating whether to open as binary or as text;
51 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
52 **kwargs
53 Additional keyword arguments to forward to all calls to `open`.
54 """
55 text_content = "abcdefghijklmnopqrstuvwxyz🙂"
56 bytes_content = uuid.uuid4().bytes
57 content_by_mode_suffix = {
58 "": text_content,
59 "t": text_content,
60 "b": bytes_content,
61 }
62 empty_content_by_mode_suffix = {
63 "": "",
64 "t": "",
65 "b": b"",
66 }
67 # To appease mypy
68 double_content_by_mode_suffix = {
69 "": text_content + text_content,
70 "t": text_content + text_content,
71 "b": bytes_content + bytes_content,
72 }
73 for mode_suffix in mode_suffixes:
74 content = content_by_mode_suffix[mode_suffix]
75 double_content = double_content_by_mode_suffix[mode_suffix]
76 # Create file with mode='x', which prohibits overwriting.
77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
78 write_buffer.write(content)
79 test_case.assertTrue(uri.exists())
80 # Check that opening with 'x' now raises, and does not modify content.
81 with test_case.assertRaises(FileExistsError):
82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
83 write_buffer.write("bad")
84 # Read the file we created and check the contents.
85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
86 test_case.assertEqual(read_buffer.read(), content)
87 # Check that we can read bytes in a loop and get EOF
88 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
89 # Seek off the end of the file and should read empty back.
90 read_buffer.seek(1024)
91 test_case.assertEqual(read_buffer.tell(), 1024)
92 content_read = read_buffer.read() # Read as much as we can.
93 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
95 # First read more than the content.
96 read_buffer.seek(0)
97 size = len(content) * 3
98 chunk_read = read_buffer.read(size)
99 test_case.assertEqual(chunk_read, content)
101 # Repeated reads should always return empty string.
102 chunk_read = read_buffer.read(size)
103 test_case.assertEqual(len(chunk_read), 0)
104 chunk_read = read_buffer.read(size)
105 test_case.assertEqual(len(chunk_read), 0)
107 # Go back to start of file and read in smaller chunks.
108 read_buffer.seek(0)
109 size = len(content) // 3
111 content_read = empty_content_by_mode_suffix[mode_suffix]
112 n_reads = 0
113 while chunk_read := read_buffer.read(size):
114 content_read += chunk_read
115 n_reads += 1
116 if n_reads > 10: # In case EOF never hits because of bug.
117 raise AssertionError(
118 f"Failed to stop reading from file after {n_reads} loops. "
119 f"Read {len(content_read)} bytes/characters. Expected {len(content)}."
120 )
121 test_case.assertEqual(content_read, content)
123 # Go back to start of file and read the entire thing.
124 read_buffer.seek(0)
125 content_read = read_buffer.read()
126 test_case.assertEqual(content_read, content)
128 # Seek off the end of the file and should read empty back.
129 # We run this check twice since in some cases the handle will
130 # cache knowledge of the file size.
131 read_buffer.seek(1024)
132 test_case.assertEqual(read_buffer.tell(), 1024)
133 content_read = read_buffer.read()
134 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
136 # Write two copies of the content, overwriting the single copy there.
137 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
138 write_buffer.write(double_content)
139 # Read again, this time use mode='r+', which reads what is there and
140 # then lets us write more; we'll use that to reset the file to one
141 # copy of the content.
142 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
143 test_case.assertEqual(rw_buffer.read(), double_content)
144 rw_buffer.seek(0)
145 rw_buffer.truncate()
146 rw_buffer.write(content)
147 rw_buffer.seek(0)
148 test_case.assertEqual(rw_buffer.read(), content)
149 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
150 test_case.assertEqual(read_buffer.read(), content)
151 # Append some more content to the file; should now have two copies.
152 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
153 append_buffer.write(content)
154 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
155 test_case.assertEqual(read_buffer.read(), double_content)
156 # Final mode to check is w+, which does read/write but truncates first.
157 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
158 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
159 rw_buffer.write(content)
160 rw_buffer.seek(0)
161 test_case.assertEqual(rw_buffer.read(), content)
162 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
163 test_case.assertEqual(read_buffer.read(), content)
164 # Remove file to make room for the next loop of tests with this URI.
165 uri.remove()
168class _GenericTestCase:
169 """Generic base class for test mixin."""
171 scheme: Optional[str] = None
172 netloc: Optional[str] = None
173 base_path: Optional[str] = None
174 path1 = "test_dir"
175 path2 = "file.txt"
177 # Because we use a mixin for tests mypy needs to understand that
178 # the unittest.TestCase methods exist.
179 # We do not inherit from unittest.TestCase because that results
180 # in the tests defined here being run as well as the tests in the
181 # test file itself. We can make those tests skip but it gives an
182 # uniformative view of how many tests are running.
183 assertEqual: Callable
184 assertNotEqual: Callable
185 assertIsNone: Callable
186 assertIn: Callable
187 assertNotIn: Callable
188 assertFalse: Callable
189 assertTrue: Callable
190 assertRaises: Callable
191 assertLogs: Callable
193 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str:
194 if self.scheme is not None:
195 if netloc is None:
196 netloc = self.netloc
197 if path.startswith("/"):
198 path = path[1:]
199 if self.base_path is not None:
200 path = f"{self.base_path}/{path}".lstrip("/")
202 return f"{self.scheme}://{netloc}/{path}"
203 else:
204 return path
207class GenericTestCase(_GenericTestCase):
208 """Test cases for generic manipulation of a `ResourcePath`"""
210 def setUp(self) -> None:
211 if self.scheme is None:
212 raise unittest.SkipTest("No scheme defined")
213 self.root = self._make_uri("")
214 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
216 def test_creation(self) -> None:
217 self.assertEqual(self.root_uri.scheme, self.scheme)
218 self.assertEqual(self.root_uri.netloc, self.netloc)
219 self.assertFalse(self.root_uri.query)
220 self.assertFalse(self.root_uri.params)
222 with self.assertRaises(ValueError):
223 ResourcePath({}) # type: ignore
225 with self.assertRaises(RuntimeError):
226 ResourcePath(self.root_uri, isTemporary=True)
228 file = self.root_uri.join("file.txt")
229 with self.assertRaises(RuntimeError):
230 ResourcePath(file, forceDirectory=True)
232 with self.assertRaises(NotImplementedError):
233 ResourcePath("unknown://netloc")
235 replaced = file.replace(fragment="frag")
236 self.assertEqual(replaced.fragment, "frag")
238 with self.assertRaises(ValueError):
239 file.replace(scheme="new")
241 self.assertNotEqual(replaced, str(replaced))
242 self.assertNotEqual(str(replaced), replaced)
244 def test_extension(self) -> None:
245 uri = ResourcePath(self._make_uri("dir/test.txt"))
246 self.assertEqual(uri.updatedExtension(None), uri)
247 self.assertEqual(uri.updatedExtension(".txt"), uri)
248 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
250 fits = uri.updatedExtension(".fits.gz")
251 self.assertEqual(fits.basename(), "test.fits.gz")
252 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
254 extensionless = self.root_uri.join("no_ext")
255 self.assertEqual(extensionless.getExtension(), "")
256 extension = extensionless.updatedExtension(".fits")
257 self.assertEqual(extension.getExtension(), ".fits")
259 def test_relative(self) -> None:
260 """Check that we can get subpaths back from two URIs"""
261 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
262 self.assertTrue(parent.isdir())
263 child = parent.join("dir1/file.txt")
265 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
267 not_child = ResourcePath("/a/b/dir1/file.txt")
268 self.assertIsNone(not_child.relative_to(parent))
269 self.assertFalse(not_child.isdir())
271 not_directory = parent.join("dir1/file2.txt")
272 self.assertIsNone(child.relative_to(not_directory))
274 # Relative URIs
275 parent = ResourcePath("a/b/", forceAbsolute=False)
276 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
277 self.assertFalse(child.scheme)
278 self.assertEqual(child.relative_to(parent), "c/d.txt")
280 # forceAbsolute=True should work even on an existing ResourcePath
281 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
283 # Absolute URI and schemeless URI
284 parent = self.root_uri.join("/a/b/c/")
285 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
287 # If the child is relative and the parent is absolute we assume
288 # that the child is a child of the parent unless it uses ".."
289 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
291 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
292 self.assertIsNone(child.relative_to(parent))
294 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
295 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
297 # Test with different netloc
298 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
299 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True)
300 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
302 # This is an absolute path so will *always* return a file URI and
303 # ignore the root parameter.
304 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True)
305 self.assertEqual(parent.geturl(), "file:///a/b/c/")
307 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True)
308 child = ResourcePath("d/e.txt", root=parent)
309 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
311 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/")))
312 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
314 # Absolute schemeless child with relative parent will always fail.
315 child = ResourcePath("d/e.txt", root="/a/b/c")
316 parent = ResourcePath("d/e.txt", forceAbsolute=False)
317 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
319 def test_parents(self) -> None:
320 """Test of splitting and parent walking."""
321 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
322 child_file = parent.join("subdir/file.txt")
323 self.assertFalse(child_file.isdir())
324 child_subdir, file = child_file.split()
325 self.assertEqual(file, "file.txt")
326 self.assertTrue(child_subdir.isdir())
327 self.assertEqual(child_file.dirname(), child_subdir)
328 self.assertEqual(child_file.basename(), file)
329 self.assertEqual(child_file.parent(), child_subdir)
330 derived_parent = child_subdir.parent()
331 self.assertEqual(derived_parent, parent)
332 self.assertTrue(derived_parent.isdir())
333 self.assertEqual(child_file.parent().parent(), parent)
335 def test_escapes(self) -> None:
336 """Special characters in file paths"""
337 src = self.root_uri.join("bbb/???/test.txt")
338 self.assertNotIn("???", src.path)
339 self.assertIn("???", src.unquoted_path)
341 file = src.updatedFile("tests??.txt")
342 self.assertNotIn("??.txt", file.path)
344 src = src.updatedFile("tests??.txt")
345 self.assertIn("??.txt", src.unquoted_path)
347 # File URI and schemeless URI
348 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
349 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
350 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
352 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
353 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
355 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
356 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
358 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
360 # dir.join() morphs into a file scheme
361 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
362 new = dir.join("test_j.txt")
363 self.assertIn("???", new.unquoted_path, f"Checking {new}")
365 new2name = "###/test??.txt"
366 new2 = dir.join(new2name)
367 self.assertIn("???", new2.unquoted_path)
368 self.assertTrue(new2.unquoted_path.endswith(new2name))
370 fdir = dir.abspath()
371 self.assertNotIn("???", fdir.path)
372 self.assertIn("???", fdir.unquoted_path)
373 self.assertEqual(fdir.scheme, self.scheme)
375 fnew2 = fdir.join(new2name)
376 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
377 self.assertNotIn("###", fnew2.path)
379 # Test that children relative to schemeless and file schemes
380 # still return the same unquoted name
381 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
382 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
383 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
384 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
386 # Check for double quoting
387 plus_path = "/a/b/c+d/"
388 with self.assertLogs(level="WARNING"):
389 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
390 self.assertEqual(uri.ospath, plus_path)
392 # Check that # is not escaped for schemeless URIs
393 hash_path = "/a/b#/c&d#xyz"
394 hpos = hash_path.rfind("#")
395 uri = ResourcePath(hash_path)
396 self.assertEqual(uri.ospath, hash_path[:hpos])
397 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
399 def test_hash(self) -> None:
400 """Test that we can store URIs in sets and as keys."""
401 uri1 = self.root_uri
402 uri2 = uri1.join("test/")
403 s = {uri1, uri2}
404 self.assertIn(uri1, s)
406 d = {uri1: "1", uri2: "2"}
407 self.assertEqual(d[uri2], "2")
409 def test_root_uri(self) -> None:
410 """Test ResourcePath.root_uri()."""
411 uri = ResourcePath(self._make_uri("a/b/c.txt"))
412 self.assertEqual(uri.root_uri().geturl(), self.root)
414 def test_join(self) -> None:
415 """Test .join method."""
416 root_str = self.root
417 root = self.root_uri
419 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
420 add_dir = root.join("b/c/d/")
421 self.assertTrue(add_dir.isdir())
422 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
424 up_relative = root.join("../b/c.txt")
425 self.assertFalse(up_relative.isdir())
426 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
428 quote_example = "hsc/payload/b&c.t@x#t"
429 needs_quote = root.join(quote_example)
430 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
432 other = ResourcePath(f"{self.root}test.txt")
433 self.assertEqual(root.join(other), other)
434 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
436 joined = ResourcePath(f"{self.root}hsc/payload/").join(
437 ResourcePath("test.qgraph", forceAbsolute=False)
438 )
439 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
441 qgraph = ResourcePath("test.qgraph") # Absolute URI
442 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph)
443 self.assertEqual(joined, qgraph)
445 def test_quoting(self) -> None:
446 """Check that quoting works."""
447 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
448 subpath = "rootdir/dir1+/file?.txt"
449 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
451 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
452 self.assertEqual(child.basename(), "file?.txt")
453 self.assertEqual(child.relativeToPathRoot, subpath)
454 self.assertIn("%", child.path)
455 self.assertEqual(child.unquoted_path, "/" + subpath)
457 def test_ordering(self) -> None:
458 """Check that greater/less comparison operators work."""
459 a = self._make_uri("a.txt")
460 b = self._make_uri("b/")
461 self.assertTrue(a < b)
462 self.assertFalse(a < a)
463 self.assertTrue(a <= b)
464 self.assertTrue(a <= a)
465 self.assertTrue(b > a)
466 self.assertFalse(b > b)
467 self.assertTrue(b >= a)
468 self.assertTrue(b >= b)
471class GenericReadWriteTestCase(_GenericTestCase):
472 """Test schemes that can read and write using concrete resources."""
474 transfer_modes = ("copy", "move")
475 testdir: Optional[str] = None
477 def setUp(self) -> None:
478 if self.scheme is None:
479 raise unittest.SkipTest("No scheme defined")
480 self.root = self._make_uri("")
481 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
483 if self.scheme == "file":
484 # Use a local tempdir because on macOS the temp dirs use symlinks
485 # so relsymlink gets quite confused.
486 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
487 else:
488 # Create random tmp directory relative to the test root.
489 self.tmpdir = self.root_uri.join(
490 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
491 forceDirectory=True,
492 )
493 self.tmpdir.mkdir()
495 def tearDown(self) -> None:
496 if self.tmpdir:
497 if self.tmpdir.isLocal:
498 removeTestTempDir(self.tmpdir.ospath)
500 def test_file(self) -> None:
501 uri = self.tmpdir.join("test.txt")
502 self.assertFalse(uri.exists(), f"{uri} should not exist")
503 self.assertTrue(uri.path.endswith("test.txt"))
505 content = "abcdefghijklmnopqrstuv\n"
506 uri.write(content.encode())
507 self.assertTrue(uri.exists(), f"{uri} should now exist")
508 self.assertEqual(uri.read().decode(), content)
509 self.assertEqual(uri.size(), len(content.encode()))
511 with self.assertRaises(FileExistsError):
512 uri.write(b"", overwrite=False)
514 # Not all backends can tell if a remove fails so we can not
515 # test that a remove of a non-existent entry is guaranteed to raise.
516 uri.remove()
517 self.assertFalse(uri.exists())
519 # Ideally the test would remove the file again and raise a
520 # FileNotFoundError. This is not reliable for remote resources
521 # and doing an explicit check before trying to remove the resource
522 # just to raise an exception is deemed an unacceptable overhead.
524 with self.assertRaises(FileNotFoundError):
525 uri.read()
527 with self.assertRaises(FileNotFoundError):
528 self.tmpdir.join("file/not/there.txt").size()
530 # Check that creating a URI from a URI returns the same thing
531 uri2 = ResourcePath(uri)
532 self.assertEqual(uri, uri2)
533 self.assertEqual(id(uri), id(uri2))
535 def test_mkdir(self) -> None:
536 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
537 newdir.mkdir()
538 self.assertTrue(newdir.exists())
539 self.assertEqual(newdir.size(), 0)
541 newfile = newdir.join("temp.txt")
542 newfile.write("Data".encode())
543 self.assertTrue(newfile.exists())
545 file = self.tmpdir.join("file.txt")
546 # Some schemes will realize that the URI is not a file and so
547 # will raise NotADirectoryError. The file scheme is more permissive
548 # and lets you write anything but will raise NotADirectoryError
549 # if a non-directory is already there. We therefore write something
550 # to the file to ensure that we trigger a portable exception.
551 file.write(b"")
552 with self.assertRaises(NotADirectoryError):
553 file.mkdir()
555 # The root should exist.
556 self.root_uri.mkdir()
557 self.assertTrue(self.root_uri.exists())
559 def test_transfer(self) -> None:
560 src = self.tmpdir.join("test.txt")
561 content = "Content is some content\nwith something to say\n\n"
562 src.write(content.encode())
564 can_move = "move" in self.transfer_modes
565 for mode in self.transfer_modes:
566 if mode == "move":
567 continue
569 dest = self.tmpdir.join(f"dest_{mode}.txt")
570 # Ensure that we get some debugging output.
571 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
572 dest.transfer_from(src, transfer=mode)
573 self.assertIn("Transferring ", "\n".join(cm.output))
574 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
576 new_content = dest.read().decode()
577 self.assertEqual(new_content, content)
579 if mode in ("symlink", "relsymlink"):
580 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
582 # If the source and destination are hardlinks of each other
583 # the transfer should work even if overwrite=False.
584 if mode in ("link", "hardlink"):
585 dest.transfer_from(src, transfer=mode)
586 else:
587 with self.assertRaises(
588 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
589 ):
590 dest.transfer_from(src, transfer=mode)
592 # Transfer again and overwrite.
593 dest.transfer_from(src, transfer=mode, overwrite=True)
595 dest.remove()
597 b = src.read()
598 self.assertEqual(b.decode(), new_content)
600 nbytes = 10
601 subset = src.read(size=nbytes)
602 self.assertEqual(len(subset), nbytes)
603 self.assertEqual(subset.decode(), content[:nbytes])
605 # Transferring to self should be okay.
606 src.transfer_from(src, "auto")
608 with self.assertRaises(ValueError):
609 src.transfer_from(src, transfer="unknown")
611 # A move transfer is special.
612 if can_move:
613 dest.transfer_from(src, transfer="move")
614 self.assertFalse(src.exists())
615 self.assertTrue(dest.exists())
616 else:
617 src.remove()
619 dest.remove()
620 with self.assertRaises(FileNotFoundError):
621 dest.transfer_from(src, "auto")
623 def test_local_transfer(self) -> None:
624 """Test we can transfer to and from local file."""
625 remote_src = self.tmpdir.join("src.json")
626 remote_src.write(b"42")
627 remote_dest = self.tmpdir.join("dest.json")
629 with ResourcePath.temporary_uri(suffix=".json") as tmp:
630 self.assertTrue(tmp.isLocal)
631 tmp.transfer_from(remote_src, transfer="auto")
632 self.assertEqual(tmp.read(), remote_src.read())
634 remote_dest.transfer_from(tmp, transfer="auto")
635 self.assertEqual(remote_dest.read(), tmp.read())
637 # Temporary (possibly remote) resource.
638 # Transfers between temporary resources.
639 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
640 # Temporary local resource.
641 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
642 remote_tmp.write(b"42")
643 if not remote_tmp.isLocal:
644 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
645 with self.assertRaises(RuntimeError):
646 # Trying to symlink a remote resource is not going
647 # to work. A hardlink could work but would rely
648 # on the local temp space being on the same
649 # filesystem as the target.
650 local_tmp.transfer_from(remote_tmp, transfer)
651 local_tmp.transfer_from(remote_tmp, "move")
652 self.assertFalse(remote_tmp.exists())
653 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
654 self.assertEqual(local_tmp.read(), remote_tmp.read())
656 # Transfer of missing remote.
657 remote_tmp.remove()
658 with self.assertRaises(FileNotFoundError):
659 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
661 def test_local(self) -> None:
662 """Check that remote resources can be made local."""
663 src = self.tmpdir.join("test.txt")
664 original_content = "Content is some content\nwith something to say\n\n"
665 src.write(original_content.encode())
667 # Run this twice to ensure use of cache in code coverage
668 # if applicable.
669 for _ in (1, 2):
670 with src.as_local() as local_uri:
671 self.assertTrue(local_uri.isLocal)
672 content = local_uri.read().decode()
673 self.assertEqual(content, original_content)
675 if src.isLocal:
676 self.assertEqual(src, local_uri)
678 with self.assertRaises(IsADirectoryError):
679 with self.root_uri.as_local() as local_uri:
680 pass
682 def test_walk(self) -> None:
683 """Walk a directory hierarchy."""
684 root = self.tmpdir.join("walk/")
686 # Look for a file that is not there
687 file = root.join("config/basic/butler.yaml")
688 found_list = list(ResourcePath.findFileResources([file]))
689 self.assertEqual(found_list[0], file)
691 # First create the files (content is irrelevant).
692 expected_files = {
693 "dir1/a.yaml",
694 "dir1/b.yaml",
695 "dir1/c.json",
696 "dir2/d.json",
697 "dir2/e.yaml",
698 }
699 expected_uris = {root.join(f) for f in expected_files}
700 for uri in expected_uris:
701 uri.write(b"")
702 self.assertTrue(uri.exists())
704 # Look for the files.
705 found = set(ResourcePath.findFileResources([root]))
706 self.assertEqual(found, expected_uris)
708 # Now solely the YAML files.
709 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
710 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
711 self.assertEqual(found, expected_yaml)
713 # Now two explicit directories and a file
714 expected = set(u for u in expected_yaml)
715 expected.add(file)
717 found = set(
718 ResourcePath.findFileResources(
719 [file, root.join("dir1/"), root.join("dir2/")],
720 file_filter=r".*\.yaml$",
721 )
722 )
723 self.assertEqual(found, expected)
725 # Group by directory -- find everything and compare it with what
726 # we expected to be there in total.
727 found_yaml = set()
728 counter = 0
729 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
730 assert not isinstance(uris, ResourcePath) # for mypy.
731 found_uris = set(uris)
732 if found_uris:
733 counter += 1
735 found_yaml.update(found_uris)
737 expected_yaml_2 = expected_yaml
738 expected_yaml_2.add(file)
739 self.assertEqual(found_yaml, expected_yaml)
740 self.assertEqual(counter, 3)
742 # Grouping but check that single files are returned in a single group
743 # at the end
744 file2 = root.join("config/templates/templates-bad.yaml")
745 found_grouped = [
746 [uri for uri in group]
747 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
748 if not isinstance(group, ResourcePath) # For mypy.
749 ]
750 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
751 self.assertEqual(list(found_grouped[1]), [file, file2])
753 with self.assertRaises(ValueError):
754 # The list forces the generator to run.
755 list(file.walk())
757 # A directory that does not exist returns nothing.
758 self.assertEqual(list(root.join("dir3/").walk()), [])
760 def test_large_walk(self) -> None:
761 # In some systems pagination is used so ensure that we can handle
762 # large numbers of files. For example S3 limits us to 1000 responses
763 # per listing call.
764 created = set()
765 counter = 1
766 n_dir1 = 1100
767 root = self.tmpdir.join("large_walk", forceDirectory=True)
768 while counter <= n_dir1:
769 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
770 new.write(f"{counter}".encode())
771 created.add(new)
772 counter += 1
773 counter = 1
774 # Put some in a subdirectory to make sure we are looking in a
775 # hierarchy.
776 n_dir2 = 100
777 subdir = root.join("subdir", forceDirectory=True)
778 while counter <= n_dir2:
779 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
780 new.write(f"{counter}".encode())
781 created.add(new)
782 counter += 1
784 found = set(ResourcePath.findFileResources([root]))
785 self.assertEqual(len(found), n_dir1 + n_dir2)
786 self.assertEqual(found, created)
788 # Again with grouping.
789 # (mypy gets upset not knowing which of the two options is being
790 # returned so add useless instance check).
791 found_list = list(
792 [uri for uri in group]
793 for group in ResourcePath.findFileResources([root], grouped=True)
794 if not isinstance(group, ResourcePath) # For mypy.
795 )
796 self.assertEqual(len(found_list), 2)
797 self.assertEqual(len(found_list[0]), n_dir1)
798 self.assertEqual(len(found_list[1]), n_dir2)
800 def test_temporary(self) -> None:
801 prefix = self.tmpdir.join("tmp", forceDirectory=True)
802 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
803 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
804 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
805 self.assertFalse(tmp.exists(), f"uri: {tmp}")
806 tmp.write(b"abcd")
807 self.assertTrue(tmp.exists(), f"uri: {tmp}")
808 self.assertTrue(tmp.isTemporary)
809 self.assertFalse(tmp.exists(), f"uri: {tmp}")
811 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
812 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
813 # Use a specified tmpdir and check it is okay for the file
814 # to not be created.
815 self.assertFalse(tmp.getExtension())
816 self.assertFalse(tmp.exists(), f"uri: {tmp}")
817 self.assertEqual(tmp.scheme, self.scheme)
818 self.assertTrue(tmp.isTemporary)
819 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
821 # Fake a directory suffix.
822 with self.assertRaises(NotImplementedError):
823 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
824 pass
826 def test_open(self) -> None:
827 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
828 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
829 _check_open(self, tmp, mode_suffixes=("", "t"))
830 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
831 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
832 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
833 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
834 _check_open(self, tmp, mode_suffixes=("b",))
835 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
837 with self.assertRaises(IsADirectoryError):
838 with self.root_uri.open():
839 pass
841 def test_mexists(self) -> None:
842 root = self.tmpdir.join("mexists/")
844 # A file that is not there.
845 file = root.join("config/basic/butler.yaml")
847 # Create some files.
848 expected_files = {
849 "dir1/a.yaml",
850 "dir1/b.yaml",
851 "dir2/e.yaml",
852 }
853 expected_uris = {root.join(f) for f in expected_files}
854 for uri in expected_uris:
855 uri.write(b"")
856 self.assertTrue(uri.exists())
857 expected_uris.add(file)
859 multi = ResourcePath.mexists(expected_uris)
861 for uri, is_there in multi.items():
862 if uri == file:
863 self.assertFalse(is_there)
864 else:
865 self.assertTrue(is_there)