Coverage for python/lsst/resources/tests.py: 9%
490 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-04 02:38 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-04 02:38 -0800
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import unittest
19import urllib.parse
20import uuid
21from typing import Any, Callable, Iterable, Optional, Union
23from lsst.resources import ResourcePath
24from lsst.resources.utils import makeTestTempDir, removeTestTempDir
26TESTDIR = os.path.abspath(os.path.dirname(__file__))
29def _check_open(
30 test_case: Union[_GenericTestCase, unittest.TestCase],
31 uri: ResourcePath,
32 *,
33 mode_suffixes: Iterable[str] = ("", "t", "b"),
34 **kwargs: Any,
35) -> None:
36 """Test an implementation of ButlerURI.open.
38 Parameters
39 ----------
40 test_case : `unittest.TestCase`
41 Test case to use for assertions.
42 uri : `ResourcePath`
43 URI to use for tests. Must point to a writeable location that is not
44 yet occupied by a file. On return, the location may point to a file
45 only if the test fails.
46 mode_suffixes : `Iterable` of `str`
47 Suffixes to pass as part of the ``mode`` argument to
48 `ResourcePath.open`, indicating whether to open as binary or as text;
49 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
50 **kwargs
51 Additional keyword arguments to forward to all calls to `open`.
52 """
53 text_content = "wxyz🙂"
54 bytes_content = uuid.uuid4().bytes
55 content_by_mode_suffix = {
56 "": text_content,
57 "t": text_content,
58 "b": bytes_content,
59 }
60 empty_content_by_mode_suffix = {
61 "": "",
62 "t": "",
63 "b": b"",
64 }
65 # To appease mypy
66 double_content_by_mode_suffix = {
67 "": text_content + text_content,
68 "t": text_content + text_content,
69 "b": bytes_content + bytes_content,
70 }
71 for mode_suffix in mode_suffixes:
72 content = content_by_mode_suffix[mode_suffix]
73 double_content = double_content_by_mode_suffix[mode_suffix]
74 # Create file with mode='x', which prohibits overwriting.
75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
76 write_buffer.write(content)
77 test_case.assertTrue(uri.exists())
78 # Check that opening with 'x' now raises, and does not modify content.
79 with test_case.assertRaises(FileExistsError):
80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
81 write_buffer.write("bad")
82 # Read the file we created and check the contents.
83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
84 test_case.assertEqual(read_buffer.read(), content)
85 # Write two copies of the content, overwriting the single copy there.
86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
87 write_buffer.write(double_content)
88 # Read again, this time use mode='r+', which reads what is there and
89 # then lets us write more; we'll use that to reset the file to one
90 # copy of the content.
91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
92 test_case.assertEqual(rw_buffer.read(), double_content)
93 rw_buffer.seek(0)
94 rw_buffer.truncate()
95 rw_buffer.write(content)
96 rw_buffer.seek(0)
97 test_case.assertEqual(rw_buffer.read(), content)
98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
99 test_case.assertEqual(read_buffer.read(), content)
100 # Append some more content to the file; should now have two copies.
101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
102 append_buffer.write(content)
103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
104 test_case.assertEqual(read_buffer.read(), double_content)
105 # Final mode to check is w+, which does read/write but truncates first.
106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
108 rw_buffer.write(content)
109 rw_buffer.seek(0)
110 test_case.assertEqual(rw_buffer.read(), content)
111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
112 test_case.assertEqual(read_buffer.read(), content)
113 # Remove file to make room for the next loop of tests with this URI.
114 uri.remove()
117class _GenericTestCase:
118 """Generic base class for test mixin."""
120 scheme: Optional[str] = None
121 netloc: Optional[str] = None
122 path1 = "test_dir"
123 path2 = "file.txt"
125 # Because we use a mixin for tests mypy needs to understand that
126 # the unittest.TestCase methods exist.
127 # We do not inherit from unittest.TestCase because that results
128 # in the tests defined here being run as well as the tests in the
129 # test file itself. We can make those tests skip but it gives an
130 # uniformative view of how many tests are running.
131 assertEqual: Callable
132 assertNotEqual: Callable
133 assertIsNone: Callable
134 assertIn: Callable
135 assertNotIn: Callable
136 assertFalse: Callable
137 assertTrue: Callable
138 assertRaises: Callable
139 assertLogs: Callable
141 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str:
142 if self.scheme is not None:
143 if netloc is None:
144 netloc = self.netloc
145 if path.startswith("/"):
146 path = path[1:]
147 return f"{self.scheme}://{netloc}/{path}"
148 else:
149 return path
152class GenericTestCase(_GenericTestCase):
153 """Test cases for generic manipulation of a `ResourcePath`"""
155 def setUp(self) -> None:
156 if self.scheme is None:
157 raise unittest.SkipTest("No scheme defined")
158 self.root = self._make_uri("")
159 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
161 def test_creation(self) -> None:
162 self.assertEqual(self.root_uri.scheme, self.scheme)
163 self.assertEqual(self.root_uri.netloc, self.netloc)
164 self.assertFalse(self.root_uri.query)
165 self.assertFalse(self.root_uri.params)
167 with self.assertRaises(ValueError):
168 ResourcePath({}) # type: ignore
170 with self.assertRaises(RuntimeError):
171 ResourcePath(self.root_uri, isTemporary=True)
173 file = self.root_uri.join("file.txt")
174 with self.assertRaises(RuntimeError):
175 ResourcePath(file, forceDirectory=True)
177 with self.assertRaises(NotImplementedError):
178 ResourcePath("unknown://netloc")
180 replaced = file.replace(fragment="frag")
181 self.assertEqual(replaced.fragment, "frag")
183 with self.assertRaises(ValueError):
184 file.replace(scheme="new")
186 self.assertNotEqual(replaced, str(replaced))
187 self.assertNotEqual(str(replaced), replaced)
189 def test_extension(self) -> None:
190 uri = ResourcePath(self._make_uri("dir/test.txt"))
191 self.assertEqual(uri.updatedExtension(None), uri)
192 self.assertEqual(uri.updatedExtension(".txt"), uri)
193 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
195 fits = uri.updatedExtension(".fits.gz")
196 self.assertEqual(fits.basename(), "test.fits.gz")
197 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
199 extensionless = self.root_uri.join("no_ext")
200 self.assertEqual(extensionless.getExtension(), "")
201 extension = extensionless.updatedExtension(".fits")
202 self.assertEqual(extension.getExtension(), ".fits")
204 def test_relative(self) -> None:
205 """Check that we can get subpaths back from two URIs"""
206 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
207 self.assertTrue(parent.isdir())
208 child = parent.join("dir1/file.txt")
210 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
212 not_child = ResourcePath("/a/b/dir1/file.txt")
213 self.assertIsNone(not_child.relative_to(parent))
214 self.assertFalse(not_child.isdir())
216 not_directory = parent.join("dir1/file2.txt")
217 self.assertIsNone(child.relative_to(not_directory))
219 # Relative URIs
220 parent = ResourcePath("a/b/", forceAbsolute=False)
221 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
222 self.assertFalse(child.scheme)
223 self.assertEqual(child.relative_to(parent), "c/d.txt")
225 # forceAbsolute=True should work even on an existing ResourcePath
226 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
228 # Absolute URI and schemeless URI
229 parent = self.root_uri.join("/a/b/c/")
230 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
232 # If the child is relative and the parent is absolute we assume
233 # that the child is a child of the parent unless it uses ".."
234 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
236 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
237 self.assertIsNone(child.relative_to(parent))
239 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
240 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
242 # Test with different netloc
243 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
244 parent = ResourcePath(self._make_uri("a", netloc="other"))
245 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
247 # Schemeless absolute child.
248 # Schemeless absolute URI is constructed using root= parameter.
249 # For now the root parameter can not be anything other than a file.
250 if self.scheme == "file":
251 parent = ResourcePath("/a/b/c", root=self.root_uri)
252 child = ResourcePath("d/e.txt", root=parent)
253 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
255 parent = ResourcePath("c/", root="/a/b/")
256 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
258 # Absolute schemeless child with relative parent will always fail.
259 parent = ResourcePath("d/e.txt", forceAbsolute=False)
260 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
262 def test_parents(self) -> None:
263 """Test of splitting and parent walking."""
264 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
265 child_file = parent.join("subdir/file.txt")
266 self.assertFalse(child_file.isdir())
267 child_subdir, file = child_file.split()
268 self.assertEqual(file, "file.txt")
269 self.assertTrue(child_subdir.isdir())
270 self.assertEqual(child_file.dirname(), child_subdir)
271 self.assertEqual(child_file.basename(), file)
272 self.assertEqual(child_file.parent(), child_subdir)
273 derived_parent = child_subdir.parent()
274 self.assertEqual(derived_parent, parent)
275 self.assertTrue(derived_parent.isdir())
276 self.assertEqual(child_file.parent().parent(), parent)
278 def test_escapes(self) -> None:
279 """Special characters in file paths"""
280 src = self.root_uri.join("bbb/???/test.txt")
281 self.assertNotIn("???", src.path)
282 self.assertIn("???", src.unquoted_path)
284 file = src.updatedFile("tests??.txt")
285 self.assertNotIn("??.txt", file.path)
287 src = src.updatedFile("tests??.txt")
288 self.assertIn("??.txt", src.unquoted_path)
290 # File URI and schemeless URI
291 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
292 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
293 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
295 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
296 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
298 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
299 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
301 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
303 # dir.join() morphs into a file scheme
304 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
305 new = dir.join("test_j.txt")
306 self.assertIn("???", new.unquoted_path, f"Checking {new}")
308 new2name = "###/test??.txt"
309 new2 = dir.join(new2name)
310 self.assertIn("???", new2.unquoted_path)
311 self.assertTrue(new2.unquoted_path.endswith(new2name))
313 fdir = dir.abspath()
314 self.assertNotIn("???", fdir.path)
315 self.assertIn("???", fdir.unquoted_path)
316 self.assertEqual(fdir.scheme, self.scheme)
318 fnew2 = fdir.join(new2name)
319 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
320 self.assertNotIn("###", fnew2.path)
322 # Test that children relative to schemeless and file schemes
323 # still return the same unquoted name
324 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
325 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
326 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
327 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
329 # Check for double quoting
330 plus_path = "/a/b/c+d/"
331 with self.assertLogs(level="WARNING"):
332 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
333 self.assertEqual(uri.ospath, plus_path)
335 # Check that # is not escaped for schemeless URIs
336 hash_path = "/a/b#/c&d#xyz"
337 hpos = hash_path.rfind("#")
338 uri = ResourcePath(hash_path)
339 self.assertEqual(uri.ospath, hash_path[:hpos])
340 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
342 def test_hash(self) -> None:
343 """Test that we can store URIs in sets and as keys."""
344 uri1 = self.root_uri
345 uri2 = uri1.join("test/")
346 s = {uri1, uri2}
347 self.assertIn(uri1, s)
349 d = {uri1: "1", uri2: "2"}
350 self.assertEqual(d[uri2], "2")
352 def test_root_uri(self) -> None:
353 """Test ResourcePath.root_uri()."""
354 uri = ResourcePath(self._make_uri("a/b/c.txt"))
355 self.assertEqual(uri.root_uri().geturl(), self.root)
357 def test_join(self) -> None:
358 """Test .join method."""
359 root_str = self.root
360 root = self.root_uri
362 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
363 add_dir = root.join("b/c/d/")
364 self.assertTrue(add_dir.isdir())
365 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
367 up_relative = root.join("../b/c.txt")
368 self.assertFalse(up_relative.isdir())
369 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
371 quote_example = "hsc/payload/b&c.t@x#t"
372 needs_quote = root.join(quote_example)
373 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
375 other = ResourcePath(f"{self.root}test.txt")
376 self.assertEqual(root.join(other), other)
377 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
379 joined = ResourcePath(f"{self.root}hsc/payload/").join(
380 ResourcePath("test.qgraph", forceAbsolute=False)
381 )
382 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
384 with self.assertRaises(ValueError):
385 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph"))
387 def test_quoting(self) -> None:
388 """Check that quoting works."""
389 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
390 subpath = "rootdir/dir1+/file?.txt"
391 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
393 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
394 self.assertEqual(child.basename(), "file?.txt")
395 self.assertEqual(child.relativeToPathRoot, subpath)
396 self.assertIn("%", child.path)
397 self.assertEqual(child.unquoted_path, "/" + subpath)
399 def test_ordering(self) -> None:
400 """Check that greater/less comparison operators work."""
401 a = self._make_uri("a.txt")
402 b = self._make_uri("b/")
403 self.assertTrue(a < b)
404 self.assertFalse(a < a)
405 self.assertTrue(a <= b)
406 self.assertTrue(a <= a)
407 self.assertTrue(b > a)
408 self.assertFalse(b > b)
409 self.assertTrue(b >= a)
410 self.assertTrue(b >= b)
413class GenericReadWriteTestCase(_GenericTestCase):
414 """Test schemes that can read and write using concrete resources."""
416 transfer_modes = ("copy", "move")
417 testdir: Optional[str] = None
419 def setUp(self) -> None:
420 if self.scheme is None:
421 raise unittest.SkipTest("No scheme defined")
422 self.root = self._make_uri("")
423 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
425 if self.scheme == "file":
426 # Use a local tempdir because on macOS the temp dirs use symlinks
427 # so relsymlink gets quite confused.
428 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
429 else:
430 # Create tmp directory relative to the test root.
431 self.tmpdir = self.root_uri.join("TESTING/")
432 self.tmpdir.mkdir()
434 def tearDown(self) -> None:
435 if self.tmpdir:
436 if self.tmpdir.isLocal:
437 removeTestTempDir(self.tmpdir.ospath)
439 def test_file(self) -> None:
440 uri = self.tmpdir.join("test.txt")
441 self.assertFalse(uri.exists(), f"{uri} should not exist")
442 self.assertTrue(uri.path.endswith("test.txt"))
444 content = "abcdefghijklmnopqrstuv\n"
445 uri.write(content.encode())
446 self.assertTrue(uri.exists(), f"{uri} should now exist")
447 self.assertEqual(uri.read().decode(), content)
448 self.assertEqual(uri.size(), len(content.encode()))
450 with self.assertRaises(FileExistsError):
451 uri.write(b"", overwrite=False)
453 # Not all backends can tell if a remove fails so we can not
454 # test that a remove of a non-existent entry is guaranteed to raise.
455 uri.remove()
456 self.assertFalse(uri.exists())
458 # Ideally the test would remove the file again and raise a
459 # FileNotFoundError. This is not reliable for remote resources
460 # and doing an explicit check before trying to remove the resource
461 # just to raise an exception is deemed an unacceptable overhead.
463 with self.assertRaises(FileNotFoundError):
464 uri.read()
466 with self.assertRaises(FileNotFoundError):
467 self.tmpdir.join("file/not/there.txt").size()
469 # Check that creating a URI from a URI returns the same thing
470 uri2 = ResourcePath(uri)
471 self.assertEqual(uri, uri2)
472 self.assertEqual(id(uri), id(uri2))
474 def test_mkdir(self) -> None:
475 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
476 newdir.mkdir()
477 self.assertTrue(newdir.exists())
478 self.assertEqual(newdir.size(), 0)
480 newfile = newdir.join("temp.txt")
481 newfile.write("Data".encode())
482 self.assertTrue(newfile.exists())
484 file = self.tmpdir.join("file.txt")
485 # Some schemes will realize that the URI is not a file and so
486 # will raise NotADirectoryError. The file scheme is more permissive
487 # and lets you write anything but will raise NotADirectoryError
488 # if a non-directory is already there. We therefore write something
489 # to the file to ensure that we trigger a portable exception.
490 file.write(b"")
491 with self.assertRaises(NotADirectoryError):
492 file.mkdir()
494 # The root should exist.
495 self.root_uri.mkdir()
496 self.assertTrue(self.root_uri.exists())
498 def test_transfer(self) -> None:
499 src = self.tmpdir.join("test.txt")
500 content = "Content is some content\nwith something to say\n\n"
501 src.write(content.encode())
503 can_move = "move" in self.transfer_modes
504 for mode in self.transfer_modes:
505 if mode == "move":
506 continue
508 dest = self.tmpdir.join(f"dest_{mode}.txt")
509 # Ensure that we get some debugging output.
510 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
511 dest.transfer_from(src, transfer=mode)
512 self.assertIn("Transferring ", "\n".join(cm.output))
513 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
515 new_content = dest.read().decode()
516 self.assertEqual(new_content, content)
518 if mode in ("symlink", "relsymlink"):
519 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
521 # If the source and destination are hardlinks of each other
522 # the transfer should work even if overwrite=False.
523 if mode in ("link", "hardlink"):
524 dest.transfer_from(src, transfer=mode)
525 else:
526 with self.assertRaises(
527 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
528 ):
529 dest.transfer_from(src, transfer=mode)
531 # Transfer again and overwrite.
532 dest.transfer_from(src, transfer=mode, overwrite=True)
534 dest.remove()
536 b = src.read()
537 self.assertEqual(b.decode(), new_content)
539 nbytes = 10
540 subset = src.read(size=nbytes)
541 self.assertEqual(len(subset), nbytes)
542 self.assertEqual(subset.decode(), content[:nbytes])
544 # Transferring to self should be okay.
545 src.transfer_from(src, "auto")
547 with self.assertRaises(ValueError):
548 src.transfer_from(src, transfer="unknown")
550 # A move transfer is special.
551 if can_move:
552 dest.transfer_from(src, transfer="move")
553 self.assertFalse(src.exists())
554 self.assertTrue(dest.exists())
555 else:
556 src.remove()
558 dest.remove()
559 with self.assertRaises(FileNotFoundError):
560 dest.transfer_from(src, "auto")
562 def test_local_transfer(self) -> None:
563 """Test we can transfer to and from local file."""
564 remote_src = self.tmpdir.join("src.json")
565 remote_src.write(b"42")
566 remote_dest = self.tmpdir.join("dest.json")
568 with ResourcePath.temporary_uri(suffix=".json") as tmp:
569 self.assertTrue(tmp.isLocal)
570 tmp.transfer_from(remote_src, transfer="auto")
571 self.assertEqual(tmp.read(), remote_src.read())
573 remote_dest.transfer_from(tmp, transfer="auto")
574 self.assertEqual(remote_dest.read(), tmp.read())
576 # Temporary (possibly remote) resource.
577 # Transfers between temporary resources.
578 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
579 # Temporary local resource.
580 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
581 remote_tmp.write(b"42")
582 if not remote_tmp.isLocal:
583 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
584 with self.assertRaises(RuntimeError):
585 # Trying to symlink a remote resource is not going
586 # to work. A hardlink could work but would rely
587 # on the local temp space being on the same
588 # filesystem as the target.
589 local_tmp.transfer_from(remote_tmp, transfer)
590 local_tmp.transfer_from(remote_tmp, "move")
591 self.assertFalse(remote_tmp.exists())
592 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
593 self.assertEqual(local_tmp.read(), remote_tmp.read())
595 # Transfer of missing remote.
596 remote_tmp.remove()
597 with self.assertRaises(FileNotFoundError):
598 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
600 def test_local(self) -> None:
601 """Check that remote resources can be made local."""
602 src = self.tmpdir.join("test.txt")
603 original_content = "Content is some content\nwith something to say\n\n"
604 src.write(original_content.encode())
606 # Run this twice to ensure use of cache in code coverage
607 # if applicable.
608 for _ in (1, 2):
609 with src.as_local() as local_uri:
610 self.assertTrue(local_uri.isLocal)
611 content = local_uri.read().decode()
612 self.assertEqual(content, original_content)
614 if src.isLocal:
615 self.assertEqual(src, local_uri)
617 with self.assertRaises(IsADirectoryError):
618 with self.root_uri.as_local() as local_uri:
619 pass
621 def test_walk(self) -> None:
622 """Walk a directory hierarchy."""
623 root = self.tmpdir.join("walk/")
625 # Look for a file that is not there
626 file = root.join("config/basic/butler.yaml")
627 found_list = list(ResourcePath.findFileResources([file]))
628 self.assertEqual(found_list[0], file)
630 # First create the files (content is irrelevant).
631 expected_files = {
632 "dir1/a.yaml",
633 "dir1/b.yaml",
634 "dir1/c.json",
635 "dir2/d.json",
636 "dir2/e.yaml",
637 }
638 expected_uris = {root.join(f) for f in expected_files}
639 for uri in expected_uris:
640 uri.write(b"")
641 self.assertTrue(uri.exists())
643 # Look for the files.
644 found = set(ResourcePath.findFileResources([root]))
645 self.assertEqual(found, expected_uris)
647 # Now solely the YAML files.
648 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
649 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
650 self.assertEqual(found, expected_yaml)
652 # Now two explicit directories and a file
653 expected = set(u for u in expected_yaml)
654 expected.add(file)
656 found = set(
657 ResourcePath.findFileResources(
658 [file, root.join("dir1/"), root.join("dir2/")],
659 file_filter=r".*\.yaml$",
660 )
661 )
662 self.assertEqual(found, expected)
664 # Group by directory -- find everything and compare it with what
665 # we expected to be there in total.
666 found_yaml = set()
667 counter = 0
668 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
669 assert not isinstance(uris, ResourcePath) # for mypy.
670 found_uris = set(uris)
671 if found_uris:
672 counter += 1
674 found_yaml.update(found_uris)
676 expected_yaml_2 = expected_yaml
677 expected_yaml_2.add(file)
678 self.assertEqual(found_yaml, expected_yaml)
679 self.assertEqual(counter, 3)
681 # Grouping but check that single files are returned in a single group
682 # at the end
683 file2 = root.join("config/templates/templates-bad.yaml")
684 found_grouped = [
685 [uri for uri in group]
686 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
687 if not isinstance(group, ResourcePath) # For mypy.
688 ]
689 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
690 self.assertEqual(list(found_grouped[1]), [file, file2])
692 with self.assertRaises(ValueError):
693 # The list forces the generator to run.
694 list(file.walk())
696 # A directory that does not exist returns nothing.
697 self.assertEqual(list(root.join("dir3/").walk()), [])
699 def test_large_walk(self) -> None:
700 # In some systems pagination is used so ensure that we can handle
701 # large numbers of files. For example S3 limits us to 1000 responses
702 # per listing call.
703 created = set()
704 counter = 1
705 n_dir1 = 1100
706 root = self.tmpdir.join("large_walk", forceDirectory=True)
707 while counter <= n_dir1:
708 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
709 new.write(f"{counter}".encode())
710 created.add(new)
711 counter += 1
712 counter = 1
713 # Put some in a subdirectory to make sure we are looking in a
714 # hierarchy.
715 n_dir2 = 100
716 subdir = root.join("subdir", forceDirectory=True)
717 while counter <= n_dir2:
718 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
719 new.write(f"{counter}".encode())
720 created.add(new)
721 counter += 1
723 found = set(ResourcePath.findFileResources([root]))
724 self.assertEqual(len(found), n_dir1 + n_dir2)
725 self.assertEqual(found, created)
727 # Again with grouping.
728 # (mypy gets upset not knowing which of the two options is being
729 # returned so add useless instance check).
730 found_list = list(
731 [uri for uri in group]
732 for group in ResourcePath.findFileResources([root], grouped=True)
733 if not isinstance(group, ResourcePath) # For mypy.
734 )
735 self.assertEqual(len(found_list), 2)
736 self.assertEqual(len(found_list[0]), n_dir1)
737 self.assertEqual(len(found_list[1]), n_dir2)
739 def test_temporary(self) -> None:
740 prefix = self.tmpdir.join("tmp", forceDirectory=True)
741 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
742 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
743 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
744 self.assertFalse(tmp.exists(), f"uri: {tmp}")
745 tmp.write(b"abcd")
746 self.assertTrue(tmp.exists(), f"uri: {tmp}")
747 self.assertTrue(tmp.isTemporary)
748 self.assertFalse(tmp.exists(), f"uri: {tmp}")
750 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
751 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
752 # Use a specified tmpdir and check it is okay for the file
753 # to not be created.
754 self.assertFalse(tmp.getExtension())
755 self.assertFalse(tmp.exists(), f"uri: {tmp}")
756 self.assertEqual(tmp.scheme, self.scheme)
757 self.assertTrue(tmp.isTemporary)
758 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
760 # Fake a directory suffix.
761 with self.assertRaises(NotImplementedError):
762 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
763 pass
765 def test_open(self) -> None:
766 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
767 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
768 _check_open(self, tmp, mode_suffixes=("", "t"))
769 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
770 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
771 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
772 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
773 _check_open(self, tmp, mode_suffixes=("b",))
774 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
776 with self.assertRaises(IsADirectoryError):
777 with self.root_uri.open():
778 pass
780 def test_mexists(self) -> None:
781 root = self.tmpdir.join("mexists/")
783 # A file that is not there.
784 file = root.join("config/basic/butler.yaml")
786 # Create some files.
787 expected_files = {
788 "dir1/a.yaml",
789 "dir1/b.yaml",
790 "dir2/e.yaml",
791 }
792 expected_uris = {root.join(f) for f in expected_files}
793 for uri in expected_uris:
794 uri.write(b"")
795 self.assertTrue(uri.exists())
796 expected_uris.add(file)
798 multi = ResourcePath.mexists(expected_uris)
800 for uri, is_there in multi.items():
801 if uri == file:
802 self.assertFalse(is_there)
803 else:
804 self.assertTrue(is_there)