Coverage for python/lsst/resources/tests.py: 9%
495 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-30 02:28 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-03-30 02:28 -0700
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from typing import Any, Callable, Iterable, Optional, Union
25from lsst.resources import ResourcePath
26from lsst.resources.utils import makeTestTempDir, removeTestTempDir
28TESTDIR = os.path.abspath(os.path.dirname(__file__))
31def _check_open(
32 test_case: Union[_GenericTestCase, unittest.TestCase],
33 uri: ResourcePath,
34 *,
35 mode_suffixes: Iterable[str] = ("", "t", "b"),
36 **kwargs: Any,
37) -> None:
38 """Test an implementation of ButlerURI.open.
40 Parameters
41 ----------
42 test_case : `unittest.TestCase`
43 Test case to use for assertions.
44 uri : `ResourcePath`
45 URI to use for tests. Must point to a writeable location that is not
46 yet occupied by a file. On return, the location may point to a file
47 only if the test fails.
48 mode_suffixes : `Iterable` of `str`
49 Suffixes to pass as part of the ``mode`` argument to
50 `ResourcePath.open`, indicating whether to open as binary or as text;
51 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
52 **kwargs
53 Additional keyword arguments to forward to all calls to `open`.
54 """
55 text_content = "wxyz🙂"
56 bytes_content = uuid.uuid4().bytes
57 content_by_mode_suffix = {
58 "": text_content,
59 "t": text_content,
60 "b": bytes_content,
61 }
62 empty_content_by_mode_suffix = {
63 "": "",
64 "t": "",
65 "b": b"",
66 }
67 # To appease mypy
68 double_content_by_mode_suffix = {
69 "": text_content + text_content,
70 "t": text_content + text_content,
71 "b": bytes_content + bytes_content,
72 }
73 for mode_suffix in mode_suffixes:
74 content = content_by_mode_suffix[mode_suffix]
75 double_content = double_content_by_mode_suffix[mode_suffix]
76 # Create file with mode='x', which prohibits overwriting.
77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
78 write_buffer.write(content)
79 test_case.assertTrue(uri.exists())
80 # Check that opening with 'x' now raises, and does not modify content.
81 with test_case.assertRaises(FileExistsError):
82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
83 write_buffer.write("bad")
84 # Read the file we created and check the contents.
85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
86 test_case.assertEqual(read_buffer.read(), content)
87 # Write two copies of the content, overwriting the single copy there.
88 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
89 write_buffer.write(double_content)
90 # Read again, this time use mode='r+', which reads what is there and
91 # then lets us write more; we'll use that to reset the file to one
92 # copy of the content.
93 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
94 test_case.assertEqual(rw_buffer.read(), double_content)
95 rw_buffer.seek(0)
96 rw_buffer.truncate()
97 rw_buffer.write(content)
98 rw_buffer.seek(0)
99 test_case.assertEqual(rw_buffer.read(), content)
100 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
101 test_case.assertEqual(read_buffer.read(), content)
102 # Append some more content to the file; should now have two copies.
103 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
104 append_buffer.write(content)
105 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
106 test_case.assertEqual(read_buffer.read(), double_content)
107 # Final mode to check is w+, which does read/write but truncates first.
108 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
109 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
110 rw_buffer.write(content)
111 rw_buffer.seek(0)
112 test_case.assertEqual(rw_buffer.read(), content)
113 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
114 test_case.assertEqual(read_buffer.read(), content)
115 # Remove file to make room for the next loop of tests with this URI.
116 uri.remove()
119class _GenericTestCase:
120 """Generic base class for test mixin."""
122 scheme: Optional[str] = None
123 netloc: Optional[str] = None
124 base_path: Optional[str] = None
125 path1 = "test_dir"
126 path2 = "file.txt"
128 # Because we use a mixin for tests mypy needs to understand that
129 # the unittest.TestCase methods exist.
130 # We do not inherit from unittest.TestCase because that results
131 # in the tests defined here being run as well as the tests in the
132 # test file itself. We can make those tests skip but it gives an
133 # uniformative view of how many tests are running.
134 assertEqual: Callable
135 assertNotEqual: Callable
136 assertIsNone: Callable
137 assertIn: Callable
138 assertNotIn: Callable
139 assertFalse: Callable
140 assertTrue: Callable
141 assertRaises: Callable
142 assertLogs: Callable
144 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str:
145 if self.scheme is not None:
146 if netloc is None:
147 netloc = self.netloc
148 if path.startswith("/"):
149 path = path[1:]
150 if self.base_path is not None:
151 path = f"{self.base_path}/{path}".lstrip("/")
153 return f"{self.scheme}://{netloc}/{path}"
154 else:
155 return path
158class GenericTestCase(_GenericTestCase):
159 """Test cases for generic manipulation of a `ResourcePath`"""
161 def setUp(self) -> None:
162 if self.scheme is None:
163 raise unittest.SkipTest("No scheme defined")
164 self.root = self._make_uri("")
165 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
167 def test_creation(self) -> None:
168 self.assertEqual(self.root_uri.scheme, self.scheme)
169 self.assertEqual(self.root_uri.netloc, self.netloc)
170 self.assertFalse(self.root_uri.query)
171 self.assertFalse(self.root_uri.params)
173 with self.assertRaises(ValueError):
174 ResourcePath({}) # type: ignore
176 with self.assertRaises(RuntimeError):
177 ResourcePath(self.root_uri, isTemporary=True)
179 file = self.root_uri.join("file.txt")
180 with self.assertRaises(RuntimeError):
181 ResourcePath(file, forceDirectory=True)
183 with self.assertRaises(NotImplementedError):
184 ResourcePath("unknown://netloc")
186 replaced = file.replace(fragment="frag")
187 self.assertEqual(replaced.fragment, "frag")
189 with self.assertRaises(ValueError):
190 file.replace(scheme="new")
192 self.assertNotEqual(replaced, str(replaced))
193 self.assertNotEqual(str(replaced), replaced)
195 def test_extension(self) -> None:
196 uri = ResourcePath(self._make_uri("dir/test.txt"))
197 self.assertEqual(uri.updatedExtension(None), uri)
198 self.assertEqual(uri.updatedExtension(".txt"), uri)
199 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
201 fits = uri.updatedExtension(".fits.gz")
202 self.assertEqual(fits.basename(), "test.fits.gz")
203 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
205 extensionless = self.root_uri.join("no_ext")
206 self.assertEqual(extensionless.getExtension(), "")
207 extension = extensionless.updatedExtension(".fits")
208 self.assertEqual(extension.getExtension(), ".fits")
210 def test_relative(self) -> None:
211 """Check that we can get subpaths back from two URIs"""
212 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
213 self.assertTrue(parent.isdir())
214 child = parent.join("dir1/file.txt")
216 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
218 not_child = ResourcePath("/a/b/dir1/file.txt")
219 self.assertIsNone(not_child.relative_to(parent))
220 self.assertFalse(not_child.isdir())
222 not_directory = parent.join("dir1/file2.txt")
223 self.assertIsNone(child.relative_to(not_directory))
225 # Relative URIs
226 parent = ResourcePath("a/b/", forceAbsolute=False)
227 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
228 self.assertFalse(child.scheme)
229 self.assertEqual(child.relative_to(parent), "c/d.txt")
231 # forceAbsolute=True should work even on an existing ResourcePath
232 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
234 # Absolute URI and schemeless URI
235 parent = self.root_uri.join("/a/b/c/")
236 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
238 # If the child is relative and the parent is absolute we assume
239 # that the child is a child of the parent unless it uses ".."
240 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
242 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
243 self.assertIsNone(child.relative_to(parent))
245 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
246 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
248 # Test with different netloc
249 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
250 parent = ResourcePath(self._make_uri("a", netloc="other"))
251 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
253 # Schemeless absolute child.
254 # Schemeless absolute URI is constructed using root= parameter.
255 # For now the root parameter can not be anything other than a file.
256 if self.scheme == "file":
257 parent = ResourcePath("/a/b/c", root=self.root_uri)
258 child = ResourcePath("d/e.txt", root=parent)
259 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
261 parent = ResourcePath("c/", root="/a/b/")
262 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
264 # Absolute schemeless child with relative parent will always fail.
265 parent = ResourcePath("d/e.txt", forceAbsolute=False)
266 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
268 def test_parents(self) -> None:
269 """Test of splitting and parent walking."""
270 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
271 child_file = parent.join("subdir/file.txt")
272 self.assertFalse(child_file.isdir())
273 child_subdir, file = child_file.split()
274 self.assertEqual(file, "file.txt")
275 self.assertTrue(child_subdir.isdir())
276 self.assertEqual(child_file.dirname(), child_subdir)
277 self.assertEqual(child_file.basename(), file)
278 self.assertEqual(child_file.parent(), child_subdir)
279 derived_parent = child_subdir.parent()
280 self.assertEqual(derived_parent, parent)
281 self.assertTrue(derived_parent.isdir())
282 self.assertEqual(child_file.parent().parent(), parent)
284 def test_escapes(self) -> None:
285 """Special characters in file paths"""
286 src = self.root_uri.join("bbb/???/test.txt")
287 self.assertNotIn("???", src.path)
288 self.assertIn("???", src.unquoted_path)
290 file = src.updatedFile("tests??.txt")
291 self.assertNotIn("??.txt", file.path)
293 src = src.updatedFile("tests??.txt")
294 self.assertIn("??.txt", src.unquoted_path)
296 # File URI and schemeless URI
297 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
298 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
299 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
301 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
302 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
304 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
305 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
307 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
309 # dir.join() morphs into a file scheme
310 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
311 new = dir.join("test_j.txt")
312 self.assertIn("???", new.unquoted_path, f"Checking {new}")
314 new2name = "###/test??.txt"
315 new2 = dir.join(new2name)
316 self.assertIn("???", new2.unquoted_path)
317 self.assertTrue(new2.unquoted_path.endswith(new2name))
319 fdir = dir.abspath()
320 self.assertNotIn("???", fdir.path)
321 self.assertIn("???", fdir.unquoted_path)
322 self.assertEqual(fdir.scheme, self.scheme)
324 fnew2 = fdir.join(new2name)
325 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
326 self.assertNotIn("###", fnew2.path)
328 # Test that children relative to schemeless and file schemes
329 # still return the same unquoted name
330 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
331 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
332 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
333 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
335 # Check for double quoting
336 plus_path = "/a/b/c+d/"
337 with self.assertLogs(level="WARNING"):
338 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
339 self.assertEqual(uri.ospath, plus_path)
341 # Check that # is not escaped for schemeless URIs
342 hash_path = "/a/b#/c&d#xyz"
343 hpos = hash_path.rfind("#")
344 uri = ResourcePath(hash_path)
345 self.assertEqual(uri.ospath, hash_path[:hpos])
346 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
348 def test_hash(self) -> None:
349 """Test that we can store URIs in sets and as keys."""
350 uri1 = self.root_uri
351 uri2 = uri1.join("test/")
352 s = {uri1, uri2}
353 self.assertIn(uri1, s)
355 d = {uri1: "1", uri2: "2"}
356 self.assertEqual(d[uri2], "2")
358 def test_root_uri(self) -> None:
359 """Test ResourcePath.root_uri()."""
360 uri = ResourcePath(self._make_uri("a/b/c.txt"))
361 self.assertEqual(uri.root_uri().geturl(), self.root)
363 def test_join(self) -> None:
364 """Test .join method."""
365 root_str = self.root
366 root = self.root_uri
368 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
369 add_dir = root.join("b/c/d/")
370 self.assertTrue(add_dir.isdir())
371 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
373 up_relative = root.join("../b/c.txt")
374 self.assertFalse(up_relative.isdir())
375 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
377 quote_example = "hsc/payload/b&c.t@x#t"
378 needs_quote = root.join(quote_example)
379 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
381 other = ResourcePath(f"{self.root}test.txt")
382 self.assertEqual(root.join(other), other)
383 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
385 joined = ResourcePath(f"{self.root}hsc/payload/").join(
386 ResourcePath("test.qgraph", forceAbsolute=False)
387 )
388 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
390 with self.assertRaises(ValueError):
391 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph"))
393 def test_quoting(self) -> None:
394 """Check that quoting works."""
395 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
396 subpath = "rootdir/dir1+/file?.txt"
397 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
399 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
400 self.assertEqual(child.basename(), "file?.txt")
401 self.assertEqual(child.relativeToPathRoot, subpath)
402 self.assertIn("%", child.path)
403 self.assertEqual(child.unquoted_path, "/" + subpath)
405 def test_ordering(self) -> None:
406 """Check that greater/less comparison operators work."""
407 a = self._make_uri("a.txt")
408 b = self._make_uri("b/")
409 self.assertTrue(a < b)
410 self.assertFalse(a < a)
411 self.assertTrue(a <= b)
412 self.assertTrue(a <= a)
413 self.assertTrue(b > a)
414 self.assertFalse(b > b)
415 self.assertTrue(b >= a)
416 self.assertTrue(b >= b)
419class GenericReadWriteTestCase(_GenericTestCase):
420 """Test schemes that can read and write using concrete resources."""
422 transfer_modes = ("copy", "move")
423 testdir: Optional[str] = None
425 def setUp(self) -> None:
426 if self.scheme is None:
427 raise unittest.SkipTest("No scheme defined")
428 self.root = self._make_uri("")
429 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
431 if self.scheme == "file":
432 # Use a local tempdir because on macOS the temp dirs use symlinks
433 # so relsymlink gets quite confused.
434 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
435 else:
436 # Create random tmp directory relative to the test root.
437 self.tmpdir = self.root_uri.join(
438 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
439 forceDirectory=True,
440 )
441 self.tmpdir.mkdir()
443 def tearDown(self) -> None:
444 if self.tmpdir:
445 if self.tmpdir.isLocal:
446 removeTestTempDir(self.tmpdir.ospath)
448 def test_file(self) -> None:
449 uri = self.tmpdir.join("test.txt")
450 self.assertFalse(uri.exists(), f"{uri} should not exist")
451 self.assertTrue(uri.path.endswith("test.txt"))
453 content = "abcdefghijklmnopqrstuv\n"
454 uri.write(content.encode())
455 self.assertTrue(uri.exists(), f"{uri} should now exist")
456 self.assertEqual(uri.read().decode(), content)
457 self.assertEqual(uri.size(), len(content.encode()))
459 with self.assertRaises(FileExistsError):
460 uri.write(b"", overwrite=False)
462 # Not all backends can tell if a remove fails so we can not
463 # test that a remove of a non-existent entry is guaranteed to raise.
464 uri.remove()
465 self.assertFalse(uri.exists())
467 # Ideally the test would remove the file again and raise a
468 # FileNotFoundError. This is not reliable for remote resources
469 # and doing an explicit check before trying to remove the resource
470 # just to raise an exception is deemed an unacceptable overhead.
472 with self.assertRaises(FileNotFoundError):
473 uri.read()
475 with self.assertRaises(FileNotFoundError):
476 self.tmpdir.join("file/not/there.txt").size()
478 # Check that creating a URI from a URI returns the same thing
479 uri2 = ResourcePath(uri)
480 self.assertEqual(uri, uri2)
481 self.assertEqual(id(uri), id(uri2))
483 def test_mkdir(self) -> None:
484 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
485 newdir.mkdir()
486 self.assertTrue(newdir.exists())
487 self.assertEqual(newdir.size(), 0)
489 newfile = newdir.join("temp.txt")
490 newfile.write("Data".encode())
491 self.assertTrue(newfile.exists())
493 file = self.tmpdir.join("file.txt")
494 # Some schemes will realize that the URI is not a file and so
495 # will raise NotADirectoryError. The file scheme is more permissive
496 # and lets you write anything but will raise NotADirectoryError
497 # if a non-directory is already there. We therefore write something
498 # to the file to ensure that we trigger a portable exception.
499 file.write(b"")
500 with self.assertRaises(NotADirectoryError):
501 file.mkdir()
503 # The root should exist.
504 self.root_uri.mkdir()
505 self.assertTrue(self.root_uri.exists())
507 def test_transfer(self) -> None:
508 src = self.tmpdir.join("test.txt")
509 content = "Content is some content\nwith something to say\n\n"
510 src.write(content.encode())
512 can_move = "move" in self.transfer_modes
513 for mode in self.transfer_modes:
514 if mode == "move":
515 continue
517 dest = self.tmpdir.join(f"dest_{mode}.txt")
518 # Ensure that we get some debugging output.
519 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
520 dest.transfer_from(src, transfer=mode)
521 self.assertIn("Transferring ", "\n".join(cm.output))
522 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
524 new_content = dest.read().decode()
525 self.assertEqual(new_content, content)
527 if mode in ("symlink", "relsymlink"):
528 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
530 # If the source and destination are hardlinks of each other
531 # the transfer should work even if overwrite=False.
532 if mode in ("link", "hardlink"):
533 dest.transfer_from(src, transfer=mode)
534 else:
535 with self.assertRaises(
536 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
537 ):
538 dest.transfer_from(src, transfer=mode)
540 # Transfer again and overwrite.
541 dest.transfer_from(src, transfer=mode, overwrite=True)
543 dest.remove()
545 b = src.read()
546 self.assertEqual(b.decode(), new_content)
548 nbytes = 10
549 subset = src.read(size=nbytes)
550 self.assertEqual(len(subset), nbytes)
551 self.assertEqual(subset.decode(), content[:nbytes])
553 # Transferring to self should be okay.
554 src.transfer_from(src, "auto")
556 with self.assertRaises(ValueError):
557 src.transfer_from(src, transfer="unknown")
559 # A move transfer is special.
560 if can_move:
561 dest.transfer_from(src, transfer="move")
562 self.assertFalse(src.exists())
563 self.assertTrue(dest.exists())
564 else:
565 src.remove()
567 dest.remove()
568 with self.assertRaises(FileNotFoundError):
569 dest.transfer_from(src, "auto")
571 def test_local_transfer(self) -> None:
572 """Test we can transfer to and from local file."""
573 remote_src = self.tmpdir.join("src.json")
574 remote_src.write(b"42")
575 remote_dest = self.tmpdir.join("dest.json")
577 with ResourcePath.temporary_uri(suffix=".json") as tmp:
578 self.assertTrue(tmp.isLocal)
579 tmp.transfer_from(remote_src, transfer="auto")
580 self.assertEqual(tmp.read(), remote_src.read())
582 remote_dest.transfer_from(tmp, transfer="auto")
583 self.assertEqual(remote_dest.read(), tmp.read())
585 # Temporary (possibly remote) resource.
586 # Transfers between temporary resources.
587 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
588 # Temporary local resource.
589 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
590 remote_tmp.write(b"42")
591 if not remote_tmp.isLocal:
592 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
593 with self.assertRaises(RuntimeError):
594 # Trying to symlink a remote resource is not going
595 # to work. A hardlink could work but would rely
596 # on the local temp space being on the same
597 # filesystem as the target.
598 local_tmp.transfer_from(remote_tmp, transfer)
599 local_tmp.transfer_from(remote_tmp, "move")
600 self.assertFalse(remote_tmp.exists())
601 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
602 self.assertEqual(local_tmp.read(), remote_tmp.read())
604 # Transfer of missing remote.
605 remote_tmp.remove()
606 with self.assertRaises(FileNotFoundError):
607 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
609 def test_local(self) -> None:
610 """Check that remote resources can be made local."""
611 src = self.tmpdir.join("test.txt")
612 original_content = "Content is some content\nwith something to say\n\n"
613 src.write(original_content.encode())
615 # Run this twice to ensure use of cache in code coverage
616 # if applicable.
617 for _ in (1, 2):
618 with src.as_local() as local_uri:
619 self.assertTrue(local_uri.isLocal)
620 content = local_uri.read().decode()
621 self.assertEqual(content, original_content)
623 if src.isLocal:
624 self.assertEqual(src, local_uri)
626 with self.assertRaises(IsADirectoryError):
627 with self.root_uri.as_local() as local_uri:
628 pass
630 def test_walk(self) -> None:
631 """Walk a directory hierarchy."""
632 root = self.tmpdir.join("walk/")
634 # Look for a file that is not there
635 file = root.join("config/basic/butler.yaml")
636 found_list = list(ResourcePath.findFileResources([file]))
637 self.assertEqual(found_list[0], file)
639 # First create the files (content is irrelevant).
640 expected_files = {
641 "dir1/a.yaml",
642 "dir1/b.yaml",
643 "dir1/c.json",
644 "dir2/d.json",
645 "dir2/e.yaml",
646 }
647 expected_uris = {root.join(f) for f in expected_files}
648 for uri in expected_uris:
649 uri.write(b"")
650 self.assertTrue(uri.exists())
652 # Look for the files.
653 found = set(ResourcePath.findFileResources([root]))
654 self.assertEqual(found, expected_uris)
656 # Now solely the YAML files.
657 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
658 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
659 self.assertEqual(found, expected_yaml)
661 # Now two explicit directories and a file
662 expected = set(u for u in expected_yaml)
663 expected.add(file)
665 found = set(
666 ResourcePath.findFileResources(
667 [file, root.join("dir1/"), root.join("dir2/")],
668 file_filter=r".*\.yaml$",
669 )
670 )
671 self.assertEqual(found, expected)
673 # Group by directory -- find everything and compare it with what
674 # we expected to be there in total.
675 found_yaml = set()
676 counter = 0
677 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
678 assert not isinstance(uris, ResourcePath) # for mypy.
679 found_uris = set(uris)
680 if found_uris:
681 counter += 1
683 found_yaml.update(found_uris)
685 expected_yaml_2 = expected_yaml
686 expected_yaml_2.add(file)
687 self.assertEqual(found_yaml, expected_yaml)
688 self.assertEqual(counter, 3)
690 # Grouping but check that single files are returned in a single group
691 # at the end
692 file2 = root.join("config/templates/templates-bad.yaml")
693 found_grouped = [
694 [uri for uri in group]
695 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
696 if not isinstance(group, ResourcePath) # For mypy.
697 ]
698 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
699 self.assertEqual(list(found_grouped[1]), [file, file2])
701 with self.assertRaises(ValueError):
702 # The list forces the generator to run.
703 list(file.walk())
705 # A directory that does not exist returns nothing.
706 self.assertEqual(list(root.join("dir3/").walk()), [])
708 def test_large_walk(self) -> None:
709 # In some systems pagination is used so ensure that we can handle
710 # large numbers of files. For example S3 limits us to 1000 responses
711 # per listing call.
712 created = set()
713 counter = 1
714 n_dir1 = 1100
715 root = self.tmpdir.join("large_walk", forceDirectory=True)
716 while counter <= n_dir1:
717 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
718 new.write(f"{counter}".encode())
719 created.add(new)
720 counter += 1
721 counter = 1
722 # Put some in a subdirectory to make sure we are looking in a
723 # hierarchy.
724 n_dir2 = 100
725 subdir = root.join("subdir", forceDirectory=True)
726 while counter <= n_dir2:
727 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
728 new.write(f"{counter}".encode())
729 created.add(new)
730 counter += 1
732 found = set(ResourcePath.findFileResources([root]))
733 self.assertEqual(len(found), n_dir1 + n_dir2)
734 self.assertEqual(found, created)
736 # Again with grouping.
737 # (mypy gets upset not knowing which of the two options is being
738 # returned so add useless instance check).
739 found_list = list(
740 [uri for uri in group]
741 for group in ResourcePath.findFileResources([root], grouped=True)
742 if not isinstance(group, ResourcePath) # For mypy.
743 )
744 self.assertEqual(len(found_list), 2)
745 self.assertEqual(len(found_list[0]), n_dir1)
746 self.assertEqual(len(found_list[1]), n_dir2)
748 def test_temporary(self) -> None:
749 prefix = self.tmpdir.join("tmp", forceDirectory=True)
750 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
751 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
752 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
753 self.assertFalse(tmp.exists(), f"uri: {tmp}")
754 tmp.write(b"abcd")
755 self.assertTrue(tmp.exists(), f"uri: {tmp}")
756 self.assertTrue(tmp.isTemporary)
757 self.assertFalse(tmp.exists(), f"uri: {tmp}")
759 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
760 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
761 # Use a specified tmpdir and check it is okay for the file
762 # to not be created.
763 self.assertFalse(tmp.getExtension())
764 self.assertFalse(tmp.exists(), f"uri: {tmp}")
765 self.assertEqual(tmp.scheme, self.scheme)
766 self.assertTrue(tmp.isTemporary)
767 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
769 # Fake a directory suffix.
770 with self.assertRaises(NotImplementedError):
771 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
772 pass
774 def test_open(self) -> None:
775 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
776 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
777 _check_open(self, tmp, mode_suffixes=("", "t"))
778 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
779 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
780 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
781 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
782 _check_open(self, tmp, mode_suffixes=("b",))
783 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
785 with self.assertRaises(IsADirectoryError):
786 with self.root_uri.open():
787 pass
789 def test_mexists(self) -> None:
790 root = self.tmpdir.join("mexists/")
792 # A file that is not there.
793 file = root.join("config/basic/butler.yaml")
795 # Create some files.
796 expected_files = {
797 "dir1/a.yaml",
798 "dir1/b.yaml",
799 "dir2/e.yaml",
800 }
801 expected_uris = {root.join(f) for f in expected_files}
802 for uri in expected_uris:
803 uri.write(b"")
804 self.assertTrue(uri.exists())
805 expected_uris.add(file)
807 multi = ResourcePath.mexists(expected_uris)
809 for uri, is_there in multi.items():
810 if uri == file:
811 self.assertFalse(is_there)
812 else:
813 self.assertTrue(is_there)