Coverage for python/lsst/resources/tests.py: 11%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import unittest
19import urllib.parse
20import uuid
21from typing import Any, Callable, Iterable, Optional, Union
23from lsst.resources import ResourcePath
24from lsst.resources.utils import makeTestTempDir, removeTestTempDir
26TESTDIR = os.path.abspath(os.path.dirname(__file__))
29def _check_open(
30 test_case: Union[_GenericTestCase, unittest.TestCase],
31 uri: ResourcePath,
32 *,
33 mode_suffixes: Iterable[str] = ("", "t", "b"),
34 **kwargs: Any,
35) -> None:
36 """Test an implementation of ButlerURI.open.
38 Parameters
39 ----------
40 test_case : `unittest.TestCase`
41 Test case to use for assertions.
42 uri : `ResourcePath`
43 URI to use for tests. Must point to a writeable location that is not
44 yet occupied by a file. On return, the location may point to a file
45 only if the test fails.
46 mode_suffixes : `Iterable` of `str`
47 Suffixes to pass as part of the ``mode`` argument to
48 `ResourcePath.open`, indicating whether to open as binary or as text;
49 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
50 **kwargs
51 Additional keyword arguments to forward to all calls to `open`.
52 """
53 text_content = "wxyz🙂"
54 bytes_content = uuid.uuid4().bytes
55 content_by_mode_suffix = {
56 "": text_content,
57 "t": text_content,
58 "b": bytes_content,
59 }
60 empty_content_by_mode_suffix = {
61 "": "",
62 "t": "",
63 "b": b"",
64 }
65 # To appease mypy
66 double_content_by_mode_suffix = {
67 "": text_content + text_content,
68 "t": text_content + text_content,
69 "b": bytes_content + bytes_content,
70 }
71 for mode_suffix in mode_suffixes:
72 content = content_by_mode_suffix[mode_suffix]
73 double_content = double_content_by_mode_suffix[mode_suffix]
74 # Create file with mode='x', which prohibits overwriting.
75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
76 write_buffer.write(content)
77 test_case.assertTrue(uri.exists())
78 # Check that opening with 'x' now raises, and does not modify content.
79 with test_case.assertRaises(FileExistsError):
80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
81 write_buffer.write("bad")
82 # Read the file we created and check the contents.
83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
84 test_case.assertEqual(read_buffer.read(), content)
85 # Write two copies of the content, overwriting the single copy there.
86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
87 write_buffer.write(double_content)
88 # Read again, this time use mode='r+', which reads what is there and
89 # then lets us write more; we'll use that to reset the file to one
90 # copy of the content.
91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
92 test_case.assertEqual(rw_buffer.read(), double_content)
93 rw_buffer.seek(0)
94 rw_buffer.truncate()
95 rw_buffer.write(content)
96 rw_buffer.seek(0)
97 test_case.assertEqual(rw_buffer.read(), content)
98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
99 test_case.assertEqual(read_buffer.read(), content)
100 # Append some more content to the file; should now have two copies.
101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
102 append_buffer.write(content)
103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
104 test_case.assertEqual(read_buffer.read(), double_content)
105 # Final mode to check is w+, which does read/write but truncates first.
106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
108 rw_buffer.write(content)
109 rw_buffer.seek(0)
110 test_case.assertEqual(rw_buffer.read(), content)
111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
112 test_case.assertEqual(read_buffer.read(), content)
113 # Remove file to make room for the next loop of tests with this URI.
114 uri.remove()
117class _GenericTestCase:
118 """Generic base class for test mixin."""
120 scheme: Optional[str] = None
121 netloc: Optional[str] = None
122 path1 = "test_dir"
123 path2 = "file.txt"
125 # Because we use a mixin for tests mypy needs to understand that
126 # the unittest.TestCase methods exist.
127 # We do not inherit from unittest.TestCase because that results
128 # in the tests defined here being run as well as the tests in the
129 # test file itself. We can make those tests skip but it gives an
130 # uniformative view of how many tests are running.
131 assertEqual: Callable
132 assertNotEqual: Callable
133 assertIsNone: Callable
134 assertIn: Callable
135 assertNotIn: Callable
136 assertFalse: Callable
137 assertTrue: Callable
138 assertRaises: Callable
139 assertLogs: Callable
141 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str:
142 if self.scheme is not None:
143 if netloc is None:
144 netloc = self.netloc
145 if path.startswith("/"):
146 path = path[1:]
147 return f"{self.scheme}://{netloc}/{path}"
148 else:
149 return path
152class GenericTestCase(_GenericTestCase):
153 """Test cases for generic manipulation of a `ResourcePath`"""
155 def setUp(self) -> None:
156 if self.scheme is None:
157 raise unittest.SkipTest("No scheme defined")
158 self.root = self._make_uri("")
159 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
161 def test_creation(self) -> None:
162 self.assertEqual(self.root_uri.scheme, self.scheme)
163 self.assertEqual(self.root_uri.netloc, self.netloc)
164 self.assertFalse(self.root_uri.query)
165 self.assertFalse(self.root_uri.params)
167 with self.assertRaises(ValueError):
168 ResourcePath({}) # type: ignore
170 with self.assertRaises(RuntimeError):
171 ResourcePath(self.root_uri, isTemporary=True)
173 file = self.root_uri.join("file.txt")
174 with self.assertRaises(RuntimeError):
175 ResourcePath(file, forceDirectory=True)
177 with self.assertRaises(NotImplementedError):
178 ResourcePath("unknown://netloc")
180 replaced = file.replace(fragment="frag")
181 self.assertEqual(replaced.fragment, "frag")
183 with self.assertRaises(ValueError):
184 file.replace(scheme="new")
186 self.assertNotEqual(replaced, str(replaced))
187 self.assertNotEqual(str(replaced), replaced)
189 def test_extension(self) -> None:
190 uri = ResourcePath(self._make_uri("dir/test.txt"))
191 self.assertEqual(uri.updatedExtension(None), uri)
192 self.assertEqual(uri.updatedExtension(".txt"), uri)
193 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
195 fits = uri.updatedExtension(".fits.gz")
196 self.assertEqual(fits.basename(), "test.fits.gz")
197 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
199 extensionless = self.root_uri.join("no_ext")
200 self.assertEqual(extensionless.getExtension(), "")
201 extension = extensionless.updatedExtension(".fits")
202 self.assertEqual(extension.getExtension(), ".fits")
204 def test_relative(self) -> None:
205 """Check that we can get subpaths back from two URIs"""
206 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
207 self.assertTrue(parent.isdir())
208 child = parent.join("dir1/file.txt")
210 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
212 not_child = ResourcePath("/a/b/dir1/file.txt")
213 self.assertIsNone(not_child.relative_to(parent))
214 self.assertFalse(not_child.isdir())
216 not_directory = parent.join("dir1/file2.txt")
217 self.assertIsNone(child.relative_to(not_directory))
219 # Relative URIs
220 parent = ResourcePath("a/b/", forceAbsolute=False)
221 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
222 self.assertFalse(child.scheme)
223 self.assertEqual(child.relative_to(parent), "c/d.txt")
225 # forceAbsolute=True should work even on an existing ResourcePath
226 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
228 # Absolute URI and schemeless URI
229 parent = self.root_uri.join("/a/b/c/")
230 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
232 # If the child is relative and the parent is absolute we assume
233 # that the child is a child of the parent unless it uses ".."
234 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
236 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
237 self.assertIsNone(child.relative_to(parent))
239 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
240 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
242 # Test with different netloc
243 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
244 parent = ResourcePath(self._make_uri("a", netloc="other"))
245 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
247 # Schemeless absolute child.
248 # Schemeless absolute URI is constructed using root= parameter.
249 # For now the root parameter can not be anything other than a file.
250 if self.scheme == "file":
251 parent = ResourcePath("/a/b/c", root=self.root_uri)
252 child = ResourcePath("d/e.txt", root=parent)
253 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
255 parent = ResourcePath("c/", root="/a/b/")
256 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
258 # Absolute schemeless child with relative parent will always fail.
259 parent = ResourcePath("d/e.txt", forceAbsolute=False)
260 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
262 def test_parents(self) -> None:
263 """Test of splitting and parent walking."""
264 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
265 child_file = parent.join("subdir/file.txt")
266 self.assertFalse(child_file.isdir())
267 child_subdir, file = child_file.split()
268 self.assertEqual(file, "file.txt")
269 self.assertTrue(child_subdir.isdir())
270 self.assertEqual(child_file.dirname(), child_subdir)
271 self.assertEqual(child_file.basename(), file)
272 self.assertEqual(child_file.parent(), child_subdir)
273 derived_parent = child_subdir.parent()
274 self.assertEqual(derived_parent, parent)
275 self.assertTrue(derived_parent.isdir())
276 self.assertEqual(child_file.parent().parent(), parent)
278 def test_escapes(self) -> None:
279 """Special characters in file paths"""
280 src = self.root_uri.join("bbb/???/test.txt")
281 self.assertNotIn("???", src.path)
282 self.assertIn("???", src.unquoted_path)
284 file = src.updatedFile("tests??.txt")
285 self.assertNotIn("??.txt", file.path)
287 src = src.updatedFile("tests??.txt")
288 self.assertIn("??.txt", src.unquoted_path)
290 # File URI and schemeless URI
291 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
292 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
293 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
295 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
296 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
298 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
299 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
301 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
303 # dir.join() morphs into a file scheme
304 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
305 new = dir.join("test_j.txt")
306 self.assertIn("???", new.unquoted_path, f"Checking {new}")
308 new2name = "###/test??.txt"
309 new2 = dir.join(new2name)
310 self.assertIn("???", new2.unquoted_path)
311 self.assertTrue(new2.unquoted_path.endswith(new2name))
313 fdir = dir.abspath()
314 self.assertNotIn("???", fdir.path)
315 self.assertIn("???", fdir.unquoted_path)
316 self.assertEqual(fdir.scheme, self.scheme)
318 fnew2 = fdir.join(new2name)
319 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
320 self.assertNotIn("###", fnew2.path)
322 # Test that children relative to schemeless and file schemes
323 # still return the same unquoted name
324 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
325 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
326 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
327 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
329 # Check for double quoting
330 plus_path = "/a/b/c+d/"
331 with self.assertLogs(level="WARNING"):
332 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
333 self.assertEqual(uri.ospath, plus_path)
335 # Check that # is not escaped for schemeless URIs
336 hash_path = "/a/b#/c&d#xyz"
337 hpos = hash_path.rfind("#")
338 uri = ResourcePath(hash_path)
339 self.assertEqual(uri.ospath, hash_path[:hpos])
340 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
342 def test_hash(self) -> None:
343 """Test that we can store URIs in sets and as keys."""
344 uri1 = self.root_uri
345 uri2 = uri1.join("test/")
346 s = {uri1, uri2}
347 self.assertIn(uri1, s)
349 d = {uri1: "1", uri2: "2"}
350 self.assertEqual(d[uri2], "2")
352 def test_root_uri(self) -> None:
353 """Test ResourcePath.root_uri()."""
354 uri = ResourcePath(self._make_uri("a/b/c.txt"))
355 self.assertEqual(uri.root_uri().geturl(), self.root)
357 def test_join(self) -> None:
358 """Test .join method."""
359 root_str = self.root
360 root = self.root_uri
362 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
363 add_dir = root.join("b/c/d/")
364 self.assertTrue(add_dir.isdir())
365 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
367 up_relative = root.join("../b/c.txt")
368 self.assertFalse(up_relative.isdir())
369 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
371 quote_example = "hsc/payload/b&c.t@x#t"
372 needs_quote = root.join(quote_example)
373 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
375 other = ResourcePath(f"{self.root}test.txt")
376 self.assertEqual(root.join(other), other)
377 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt")
379 joined = ResourcePath(f"{self.root}hsc/payload/").join(
380 ResourcePath("test.qgraph", forceAbsolute=False)
381 )
382 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
384 with self.assertRaises(ValueError):
385 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph"))
387 def test_quoting(self) -> None:
388 """Check that quoting works."""
389 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
390 subpath = "rootdir/dir1+/file?.txt"
391 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
393 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
394 self.assertEqual(child.basename(), "file?.txt")
395 self.assertEqual(child.relativeToPathRoot, subpath)
396 self.assertIn("%", child.path)
397 self.assertEqual(child.unquoted_path, "/" + subpath)
400class GenericReadWriteTestCase(_GenericTestCase):
401 """Test schemes that can read and write using concrete resources."""
403 transfer_modes = ("copy", "move")
404 testdir: Optional[str] = None
406 def setUp(self) -> None:
407 if self.scheme is None:
408 raise unittest.SkipTest("No scheme defined")
409 self.root = self._make_uri("")
410 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
412 if self.scheme == "file":
413 # Use a local tempdir because on macOS the temp dirs use symlinks
414 # so relsymlink gets quite confused.
415 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir))
416 else:
417 # Create tmp directory relative to the test root.
418 self.tmpdir = self.root_uri.join("TESTING/")
419 self.tmpdir.mkdir()
421 def tearDown(self) -> None:
422 if self.tmpdir:
423 if self.tmpdir.isLocal:
424 removeTestTempDir(self.tmpdir.ospath)
426 def test_file(self) -> None:
427 uri = self.tmpdir.join("test.txt")
428 self.assertFalse(uri.exists(), f"{uri} should not exist")
429 self.assertTrue(uri.path.endswith("test.txt"))
431 content = "abcdefghijklmnopqrstuv\n"
432 uri.write(content.encode())
433 self.assertTrue(uri.exists(), f"{uri} should now exist")
434 self.assertEqual(uri.read().decode(), content)
435 self.assertEqual(uri.size(), len(content.encode()))
437 with self.assertRaises(FileExistsError):
438 uri.write(b"", overwrite=False)
440 # Not all backends can tell if a remove fails so we can not
441 # test that a remove of a non-existent entry is guaranteed to raise.
442 uri.remove()
443 self.assertFalse(uri.exists())
445 # Ideally the test would remove the file again and raise a
446 # FileNotFoundError. This is not reliable for remote resources
447 # and doing an explicit check before trying to remove the resource
448 # just to raise an exception is deemed an unacceptable overhead.
450 with self.assertRaises(FileNotFoundError):
451 uri.read()
453 with self.assertRaises(FileNotFoundError):
454 self.tmpdir.join("file/not/there.txt").size()
456 # Check that creating a URI from a URI returns the same thing
457 uri2 = ResourcePath(uri)
458 self.assertEqual(uri, uri2)
459 self.assertEqual(id(uri), id(uri2))
461 def test_mkdir(self) -> None:
462 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
463 newdir.mkdir()
464 self.assertTrue(newdir.exists())
465 self.assertEqual(newdir.size(), 0)
467 newfile = newdir.join("temp.txt")
468 newfile.write("Data".encode())
469 self.assertTrue(newfile.exists())
471 file = self.tmpdir.join("file.txt")
472 # Some schemes will realize that the URI is not a file and so
473 # will raise NotADirectoryError. The file scheme is more permissive
474 # and lets you write anything but will raise NotADirectoryError
475 # if a non-directory is already there. We therefore write something
476 # to the file to ensure that we trigger a portable exception.
477 file.write(b"")
478 with self.assertRaises(NotADirectoryError):
479 file.mkdir()
481 # The root should exist.
482 self.root_uri.mkdir()
483 self.assertTrue(self.root_uri.exists())
485 def test_transfer(self) -> None:
486 src = self.tmpdir.join("test.txt")
487 content = "Content is some content\nwith something to say\n\n"
488 src.write(content.encode())
490 can_move = "move" in self.transfer_modes
491 for mode in self.transfer_modes:
492 if mode == "move":
493 continue
495 dest = self.tmpdir.join(f"dest_{mode}.txt")
496 # Ensure that we get some debugging output.
497 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
498 dest.transfer_from(src, transfer=mode)
499 self.assertIn("Transferring ", "\n".join(cm.output))
500 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
502 new_content = dest.read().decode()
503 self.assertEqual(new_content, content)
505 if mode in ("symlink", "relsymlink"):
506 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
508 # If the source and destination are hardlinks of each other
509 # the transfer should work even if overwrite=False.
510 if mode in ("link", "hardlink"):
511 dest.transfer_from(src, transfer=mode)
512 else:
513 with self.assertRaises(
514 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
515 ):
516 dest.transfer_from(src, transfer=mode)
518 # Transfer again and overwrite.
519 dest.transfer_from(src, transfer=mode, overwrite=True)
521 dest.remove()
523 b = src.read()
524 self.assertEqual(b.decode(), new_content)
526 nbytes = 10
527 subset = src.read(size=nbytes)
528 self.assertEqual(len(subset), nbytes)
529 self.assertEqual(subset.decode(), content[:nbytes])
531 # Transferring to self should be okay.
532 src.transfer_from(src, "auto")
534 with self.assertRaises(ValueError):
535 src.transfer_from(src, transfer="unknown")
537 # A move transfer is special.
538 if can_move:
539 dest.transfer_from(src, transfer="move")
540 self.assertFalse(src.exists())
541 self.assertTrue(dest.exists())
542 else:
543 src.remove()
545 dest.remove()
546 with self.assertRaises(FileNotFoundError):
547 dest.transfer_from(src, "auto")
549 def test_local_transfer(self) -> None:
550 """Test we can transfer to and from local file."""
551 remote_src = self.tmpdir.join("src.json")
552 remote_src.write(b"42")
553 remote_dest = self.tmpdir.join("dest.json")
555 with ResourcePath.temporary_uri(suffix=".json") as tmp:
556 self.assertTrue(tmp.isLocal)
557 tmp.transfer_from(remote_src, transfer="auto")
558 self.assertEqual(tmp.read(), remote_src.read())
560 remote_dest.transfer_from(tmp, transfer="auto")
561 self.assertEqual(remote_dest.read(), tmp.read())
563 # Temporary (possibly remote) resource.
564 # Transfers between temporary resources.
565 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp:
566 # Temporary local resource.
567 with ResourcePath.temporary_uri(suffix=".json") as local_tmp:
568 remote_tmp.write(b"42")
569 if not remote_tmp.isLocal:
570 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
571 with self.assertRaises(RuntimeError):
572 # Trying to symlink a remote resource is not going
573 # to work. A hardlink could work but would rely
574 # on the local temp space being on the same
575 # filesystem as the target.
576 local_tmp.transfer_from(remote_tmp, transfer)
577 local_tmp.transfer_from(remote_tmp, "move")
578 self.assertFalse(remote_tmp.exists())
579 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
580 self.assertEqual(local_tmp.read(), remote_tmp.read())
582 # Transfer of missing remote.
583 remote_tmp.remove()
584 with self.assertRaises(FileNotFoundError):
585 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
587 def test_local(self) -> None:
588 """Check that remote resources can be made local."""
589 src = self.tmpdir.join("test.txt")
590 original_content = "Content is some content\nwith something to say\n\n"
591 src.write(original_content.encode())
593 # Run this twice to ensure use of cache in code coverage
594 # if applicable.
595 for _ in (1, 2):
596 with src.as_local() as local_uri:
597 self.assertTrue(local_uri.isLocal)
598 content = local_uri.read().decode()
599 self.assertEqual(content, original_content)
601 if src.isLocal:
602 self.assertEqual(src, local_uri)
604 with self.assertRaises(IsADirectoryError):
605 with self.root_uri.as_local() as local_uri:
606 pass
608 def test_walk(self) -> None:
609 """Walk a directory hierarchy."""
610 root = self.tmpdir.join("walk/")
612 # Look for a file that is not there
613 file = root.join("config/basic/butler.yaml")
614 found_list = list(ResourcePath.findFileResources([file]))
615 self.assertEqual(found_list[0], file)
617 # First create the files (content is irrelevant).
618 expected_files = {
619 "dir1/a.yaml",
620 "dir1/b.yaml",
621 "dir1/c.json",
622 "dir2/d.json",
623 "dir2/e.yaml",
624 }
625 expected_uris = {root.join(f) for f in expected_files}
626 for uri in expected_uris:
627 uri.write(b"")
628 self.assertTrue(uri.exists())
630 # Look for the files.
631 found = set(ResourcePath.findFileResources([root]))
632 self.assertEqual(found, expected_uris)
634 # Now solely the YAML files.
635 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
636 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
637 self.assertEqual(found, expected_yaml)
639 # Now two explicit directories and a file
640 expected = set(u for u in expected_yaml)
641 expected.add(file)
643 found = set(
644 ResourcePath.findFileResources(
645 [file, root.join("dir1/"), root.join("dir2/")],
646 file_filter=r".*\.yaml$",
647 )
648 )
649 self.assertEqual(found, expected)
651 # Group by directory -- find everything and compare it with what
652 # we expected to be there in total.
653 found_yaml = set()
654 counter = 0
655 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
656 assert not isinstance(uris, ResourcePath) # for mypy.
657 found_uris = set(uris)
658 if found_uris:
659 counter += 1
661 found_yaml.update(found_uris)
663 expected_yaml_2 = expected_yaml
664 expected_yaml_2.add(file)
665 self.assertEqual(found_yaml, expected_yaml)
666 self.assertEqual(counter, 3)
668 # Grouping but check that single files are returned in a single group
669 # at the end
670 file2 = root.join("config/templates/templates-bad.yaml")
671 found_grouped = [
672 [uri for uri in group]
673 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
674 if not isinstance(group, ResourcePath) # For mypy.
675 ]
676 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
677 self.assertEqual(list(found_grouped[1]), [file, file2])
679 with self.assertRaises(ValueError):
680 # The list forces the generator to run.
681 list(file.walk())
683 # A directory that does not exist returns nothing.
684 self.assertEqual(list(root.join("dir3/").walk()), [])
686 def test_large_walk(self) -> None:
687 # In some systems pagination is used so ensure that we can handle
688 # large numbers of files. For example S3 limits us to 1000 responses
689 # per listing call.
690 created = set()
691 counter = 1
692 n_dir1 = 1100
693 root = self.tmpdir.join("large_walk", forceDirectory=True)
694 while counter <= n_dir1:
695 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
696 new.write(f"{counter}".encode())
697 created.add(new)
698 counter += 1
699 counter = 1
700 # Put some in a subdirectory to make sure we are looking in a
701 # hierarchy.
702 n_dir2 = 100
703 subdir = root.join("subdir", forceDirectory=True)
704 while counter <= n_dir2:
705 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
706 new.write(f"{counter}".encode())
707 created.add(new)
708 counter += 1
710 found = set(ResourcePath.findFileResources([root]))
711 self.assertEqual(len(found), n_dir1 + n_dir2)
712 self.assertEqual(found, created)
714 # Again with grouping.
715 # (mypy gets upset not knowing which of the two options is being
716 # returned so add useless instance check).
717 found_list = list(
718 [uri for uri in group]
719 for group in ResourcePath.findFileResources([root], grouped=True)
720 if not isinstance(group, ResourcePath) # For mypy.
721 )
722 self.assertEqual(len(found_list), 2)
723 self.assertEqual(len(found_list[0]), n_dir1)
724 self.assertEqual(len(found_list[1]), n_dir2)
726 def test_temporary(self) -> None:
727 prefix = self.tmpdir.join("tmp", forceDirectory=True)
728 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
729 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
730 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
731 self.assertFalse(tmp.exists(), f"uri: {tmp}")
732 tmp.write(b"abcd")
733 self.assertTrue(tmp.exists(), f"uri: {tmp}")
734 self.assertTrue(tmp.isTemporary)
735 self.assertFalse(tmp.exists(), f"uri: {tmp}")
737 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
738 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
739 # Use a specified tmpdir and check it is okay for the file
740 # to not be created.
741 self.assertFalse(tmp.getExtension())
742 self.assertFalse(tmp.exists(), f"uri: {tmp}")
743 self.assertEqual(tmp.scheme, self.scheme)
744 self.assertTrue(tmp.isTemporary)
745 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
747 # Fake a directory suffix.
748 with self.assertRaises(NotImplementedError):
749 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
750 pass
752 def test_open(self) -> None:
753 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
754 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
755 _check_open(self, tmp, mode_suffixes=("", "t"))
756 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
757 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
758 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
759 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
760 _check_open(self, tmp, mode_suffixes=("b",))
761 _check_open(self, tmp, mode_suffixes=("b"), prefer_file_temporary=True)
763 with self.assertRaises(IsADirectoryError):
764 with self.root_uri.open():
765 pass
767 def test_mexists(self) -> None:
768 root = self.tmpdir.join("mexists/")
770 # A file that is not there.
771 file = root.join("config/basic/butler.yaml")
773 # Create some files.
774 expected_files = {
775 "dir1/a.yaml",
776 "dir1/b.yaml",
777 "dir2/e.yaml",
778 }
779 expected_uris = {root.join(f) for f in expected_files}
780 for uri in expected_uris:
781 uri.write(b"")
782 self.assertTrue(uri.exists())
783 expected_uris.add(file)
785 multi = ResourcePath.mexists(expected_uris)
787 for uri, is_there in multi.items():
788 if uri == file:
789 self.assertFalse(is_there)
790 else:
791 self.assertTrue(is_there)