Coverage for python/lsst/resources/tests.py: 7%
545 statements
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 11:14 +0000
« prev ^ index » next coverage.py v7.4.1, created at 2024-02-01 11:14 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
11from __future__ import annotations
13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"]
15import logging
16import os
17import pathlib
18import random
19import string
20import unittest
21import urllib.parse
22import uuid
23from collections.abc import Iterable
24from typing import TYPE_CHECKING, Any
26from lsst.resources import ResourcePath
27from lsst.resources.utils import makeTestTempDir, removeTestTempDir
29TESTDIR = os.path.abspath(os.path.dirname(__file__))
32def _check_open(
33 test_case: _GenericTestCase | unittest.TestCase,
34 uri: ResourcePath,
35 *,
36 mode_suffixes: Iterable[str] = ("", "t", "b"),
37 **kwargs: Any,
38) -> None:
39 """Test an implementation of ButlerURI.open.
41 Parameters
42 ----------
43 test_case : `unittest.TestCase`
44 Test case to use for assertions.
45 uri : `ResourcePath`
46 URI to use for tests. Must point to a writeable location that is not
47 yet occupied by a file. On return, the location may point to a file
48 only if the test fails.
49 mode_suffixes : `~collections.abc.Iterable` of `str`
50 Suffixes to pass as part of the ``mode`` argument to
51 `ResourcePath.open`, indicating whether to open as binary or as text;
52 the only permitted elements are ``""``, ``"t"``, and ``"b"`.
53 **kwargs
54 Additional keyword arguments to forward to all calls to `open`.
55 """
56 text_content = "abcdefghijklmnopqrstuvwxyz🙂"
57 bytes_content = uuid.uuid4().bytes
58 content_by_mode_suffix = {
59 "": text_content,
60 "t": text_content,
61 "b": bytes_content,
62 }
63 empty_content_by_mode_suffix = {
64 "": "",
65 "t": "",
66 "b": b"",
67 }
68 # To appease mypy
69 double_content_by_mode_suffix = {
70 "": text_content + text_content,
71 "t": text_content + text_content,
72 "b": bytes_content + bytes_content,
73 }
74 for mode_suffix in mode_suffixes:
75 content = content_by_mode_suffix[mode_suffix]
76 double_content = double_content_by_mode_suffix[mode_suffix]
77 # Create file with mode='x', which prohibits overwriting.
78 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
79 write_buffer.write(content)
80 test_case.assertTrue(uri.exists())
81 # Check that opening with 'x' now raises, and does not modify content.
82 with test_case.assertRaises(FileExistsError):
83 with uri.open("x" + mode_suffix, **kwargs) as write_buffer:
84 write_buffer.write("bad")
85 # Read the file we created and check the contents.
86 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
87 test_case.assertEqual(read_buffer.read(), content)
88 # Check that we can read bytes in a loop and get EOF
89 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
90 # Seek off the end of the file and should read empty back.
91 read_buffer.seek(1024)
92 test_case.assertEqual(read_buffer.tell(), 1024)
93 content_read = read_buffer.read() # Read as much as we can.
94 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
96 # First read more than the content.
97 read_buffer.seek(0)
98 size = len(content) * 3
99 chunk_read = read_buffer.read(size)
100 test_case.assertEqual(chunk_read, content)
102 # Repeated reads should always return empty string.
103 chunk_read = read_buffer.read(size)
104 test_case.assertEqual(len(chunk_read), 0)
105 chunk_read = read_buffer.read(size)
106 test_case.assertEqual(len(chunk_read), 0)
108 # Go back to start of file and read in smaller chunks.
109 read_buffer.seek(0)
110 size = len(content) // 3
112 content_read = empty_content_by_mode_suffix[mode_suffix]
113 n_reads = 0
114 while chunk_read := read_buffer.read(size):
115 content_read += chunk_read
116 n_reads += 1
117 if n_reads > 10: # In case EOF never hits because of bug.
118 raise AssertionError(
119 f"Failed to stop reading from file after {n_reads} loops. "
120 f"Read {len(content_read)} bytes/characters. Expected {len(content)}."
121 )
122 test_case.assertEqual(content_read, content)
124 # Go back to start of file and read the entire thing.
125 read_buffer.seek(0)
126 content_read = read_buffer.read()
127 test_case.assertEqual(content_read, content)
129 # Seek off the end of the file and should read empty back.
130 # We run this check twice since in some cases the handle will
131 # cache knowledge of the file size.
132 read_buffer.seek(1024)
133 test_case.assertEqual(read_buffer.tell(), 1024)
134 content_read = read_buffer.read()
135 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.")
137 # Write two copies of the content, overwriting the single copy there.
138 with uri.open("w" + mode_suffix, **kwargs) as write_buffer:
139 write_buffer.write(double_content)
140 # Read again, this time use mode='r+', which reads what is there and
141 # then lets us write more; we'll use that to reset the file to one
142 # copy of the content.
143 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer:
144 test_case.assertEqual(rw_buffer.read(), double_content)
145 rw_buffer.seek(0)
146 rw_buffer.truncate()
147 rw_buffer.write(content)
148 rw_buffer.seek(0)
149 test_case.assertEqual(rw_buffer.read(), content)
150 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
151 test_case.assertEqual(read_buffer.read(), content)
152 # Append some more content to the file; should now have two copies.
153 with uri.open("a" + mode_suffix, **kwargs) as append_buffer:
154 append_buffer.write(content)
155 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
156 test_case.assertEqual(read_buffer.read(), double_content)
157 # Final mode to check is w+, which does read/write but truncates first.
158 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer:
159 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix])
160 rw_buffer.write(content)
161 rw_buffer.seek(0)
162 test_case.assertEqual(rw_buffer.read(), content)
163 with uri.open("r" + mode_suffix, **kwargs) as read_buffer:
164 test_case.assertEqual(read_buffer.read(), content)
165 # Remove file to make room for the next loop of tests with this URI.
166 uri.remove()
169if TYPE_CHECKING:
171 class TestCaseMixin(unittest.TestCase):
172 """Base class for mixin test classes that use TestCase methods."""
174 pass
176else:
178 class TestCaseMixin:
179 """Do-nothing definition of mixin base class for regular execution."""
181 pass
184class _GenericTestCase(TestCaseMixin):
185 """Generic base class for test mixin."""
187 scheme: str | None = None
188 netloc: str | None = None
189 base_path: str | None = None
190 path1 = "test_dir"
191 path2 = "file.txt"
193 def _make_uri(self, path: str, netloc: str | None = None) -> str:
194 if self.scheme is not None:
195 if netloc is None:
196 netloc = self.netloc
197 if path.startswith("/"):
198 path = path[1:]
199 if self.base_path is not None:
200 path = f"{self.base_path}/{path}".lstrip("/")
202 return f"{self.scheme}://{netloc}/{path}"
203 else:
204 return path
207class GenericTestCase(_GenericTestCase):
208 """Test cases for generic manipulation of a `ResourcePath`."""
210 def setUp(self) -> None:
211 if self.scheme is None:
212 raise unittest.SkipTest("No scheme defined")
213 self.root = self._make_uri("")
214 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
216 def test_creation(self) -> None:
217 self.assertEqual(self.root_uri.scheme, self.scheme)
218 self.assertEqual(self.root_uri.netloc, self.netloc)
219 self.assertFalse(self.root_uri.query)
220 self.assertFalse(self.root_uri.params)
222 with self.assertRaises(ValueError):
223 ResourcePath({}) # type: ignore
225 with self.assertRaises(RuntimeError):
226 ResourcePath(self.root_uri, isTemporary=True)
228 file = self.root_uri.join("file.txt", forceDirectory=False)
229 with self.assertRaises(RuntimeError):
230 ResourcePath(file, forceDirectory=True)
232 file = self.root_uri.join("file.txt")
233 file_as_dir = ResourcePath(file, forceDirectory=True)
234 self.assertTrue(file_as_dir.isdir())
236 dir = self._make_uri("a/b/c/")
237 with self.assertRaises(ValueError):
238 ResourcePath(dir, forceDirectory=False)
240 with self.assertRaises(NotImplementedError):
241 ResourcePath("unknown://netloc")
243 replaced = file.replace(fragment="frag")
244 self.assertEqual(replaced.fragment, "frag")
246 with self.assertRaises(ValueError):
247 file.replace(scheme="new")
249 self.assertNotEqual(replaced, str(replaced))
250 self.assertNotEqual(str(replaced), replaced)
252 def test_extension(self) -> None:
253 uri = ResourcePath(self._make_uri("dir/test.txt"))
254 self.assertEqual(uri.updatedExtension(None), uri)
255 self.assertEqual(uri.updatedExtension(".txt"), uri)
256 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri))
258 fits = uri.updatedExtension(".fits.gz")
259 self.assertEqual(fits.basename(), "test.fits.gz")
260 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg")
262 extensionless = self.root_uri.join("no_ext")
263 self.assertEqual(extensionless.getExtension(), "")
264 extension = extensionless.updatedExtension(".fits")
265 self.assertEqual(extension.getExtension(), ".fits")
267 uri = ResourcePath("test.txt", forceAbsolute=False)
268 self.assertEqual(uri.getExtension(), ".txt")
269 uri = ResourcePath(self._make_uri("dir.1/dir.2/test.txt"), forceDirectory=False)
270 self.assertEqual(uri.getExtension(), ".txt")
271 uri = ResourcePath(self._make_uri("dir.1/dir.2/"), forceDirectory=True)
272 self.assertEqual(uri.getExtension(), ".2")
273 uri = ResourcePath(self._make_uri("dir.1/dir/"), forceDirectory=True)
274 self.assertEqual(uri.getExtension(), "")
276 def test_relative(self) -> None:
277 """Check that we can get subpaths back from two URIs."""
278 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True)
279 self.assertTrue(parent.isdir())
280 child = parent.join("dir1/file.txt")
282 self.assertEqual(child.relative_to(parent), "dir1/file.txt")
284 not_child = ResourcePath("/a/b/dir1/file.txt")
285 self.assertIsNone(not_child.relative_to(parent))
286 self.assertFalse(not_child.isdir())
288 not_directory = parent.join("dir1/file2.txt")
289 self.assertIsNone(child.relative_to(not_directory))
291 # Relative URIs
292 parent = ResourcePath("a/b/", forceAbsolute=False)
293 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False)
294 self.assertFalse(child.scheme)
295 self.assertEqual(child.relative_to(parent), "c/d.txt")
297 # forceAbsolute=True should work even on an existing ResourcePath
298 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute())
300 # Absolute URI and schemeless URI
301 parent = self.root_uri.join("/a/b/c/")
302 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
304 # If the child is relative and the parent is absolute we assume
305 # that the child is a child of the parent unless it uses ".."
306 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})")
308 child = ResourcePath("../e/f/g.txt", forceAbsolute=False)
309 self.assertIsNone(child.relative_to(parent))
311 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False)
312 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
314 # Test with different netloc
315 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host"))
316 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True)
317 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
319 # This is an absolute path so will *always* return a file URI and
320 # ignore the root parameter.
321 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True)
322 self.assertEqual(parent.geturl(), "file:///a/b/c/")
324 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True)
325 child = ResourcePath("d/e.txt", root=parent)
326 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
328 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/")))
329 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})")
331 # Absolute schemeless child with relative parent will always fail.
332 child = ResourcePath("d/e.txt", root="/a/b/c")
333 parent = ResourcePath("d/e.txt", forceAbsolute=False)
334 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})")
336 def test_parents(self) -> None:
337 """Test of splitting and parent walking."""
338 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True)
339 child_file = parent.join("subdir/file.txt")
340 self.assertFalse(child_file.isdir())
341 child_subdir, file = child_file.split()
342 self.assertEqual(file, "file.txt")
343 self.assertTrue(child_subdir.isdir())
344 self.assertEqual(child_file.dirname(), child_subdir)
345 self.assertEqual(child_file.basename(), file)
346 self.assertEqual(child_file.parent(), child_subdir)
347 derived_parent = child_subdir.parent()
348 self.assertEqual(derived_parent, parent)
349 self.assertTrue(derived_parent.isdir())
350 self.assertEqual(child_file.parent().parent(), parent)
351 self.assertEqual(child_subdir.dirname(), child_subdir)
353 def test_escapes(self) -> None:
354 """Special characters in file paths."""
355 src = self.root_uri.join("bbb/???/test.txt")
356 self.assertNotIn("???", src.path)
357 self.assertIn("???", src.unquoted_path)
359 file = src.updatedFile("tests??.txt")
360 self.assertNotIn("??.txt", file.path)
362 src = src.updatedFile("tests??.txt")
363 self.assertIn("??.txt", src.unquoted_path)
365 # File URI and schemeless URI
366 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/")))
367 child = ResourcePath("e/f/g.txt", forceAbsolute=False)
368 self.assertEqual(child.relative_to(parent), "e/f/g.txt")
370 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False)
371 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
373 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")))
374 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt")
376 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt")
378 # dir.join() morphs into a file scheme
379 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/")))
380 new = dir.join("test_j.txt")
381 self.assertIn("???", new.unquoted_path, f"Checking {new}")
383 new2name = "###/test??.txt"
384 new2 = dir.join(new2name)
385 self.assertIn("???", new2.unquoted_path)
386 self.assertTrue(new2.unquoted_path.endswith(new2name))
388 fdir = dir.abspath()
389 self.assertNotIn("???", fdir.path)
390 self.assertIn("???", fdir.unquoted_path)
391 self.assertEqual(fdir.scheme, self.scheme)
393 fnew2 = fdir.join(new2name)
394 self.assertTrue(fnew2.unquoted_path.endswith(new2name))
395 self.assertNotIn("###", fnew2.path)
397 # Test that children relative to schemeless and file schemes
398 # still return the same unquoted name
399 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})")
400 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})")
401 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})")
402 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})")
404 # Check for double quoting
405 plus_path = "/a/b/c+d/"
406 with self.assertLogs(level="WARNING"):
407 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True)
408 self.assertEqual(uri.ospath, plus_path)
410 # Check that # is not escaped for schemeless URIs
411 hash_path = "/a/b#/c&d#xyz"
412 hpos = hash_path.rfind("#")
413 uri = ResourcePath(hash_path)
414 self.assertEqual(uri.ospath, hash_path[:hpos])
415 self.assertEqual(uri.fragment, hash_path[hpos + 1 :])
417 def test_hash(self) -> None:
418 """Test that we can store URIs in sets and as keys."""
419 uri1 = self.root_uri
420 uri2 = uri1.join("test/")
421 s = {uri1, uri2}
422 self.assertIn(uri1, s)
424 d = {uri1: "1", uri2: "2"}
425 self.assertEqual(d[uri2], "2")
427 def test_root_uri(self) -> None:
428 """Test ResourcePath.root_uri()."""
429 uri = ResourcePath(self._make_uri("a/b/c.txt"))
430 self.assertEqual(uri.root_uri().geturl(), self.root)
432 def test_join(self) -> None:
433 """Test .join method."""
434 root_str = self.root
435 root = self.root_uri
437 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt")
438 add_dir = root.join("b/c/d/")
439 self.assertTrue(add_dir.isdir())
440 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/")
442 up_relative = root.join("../b/c.txt")
443 self.assertFalse(up_relative.isdir())
444 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt")
446 quote_example = "hsc/payload/b&c.t@x#t"
447 needs_quote = root.join(quote_example)
448 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example)
450 other = ResourcePath(f"{self.root}test.txt")
451 self.assertEqual(root.join(other), other)
452 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}test.txt/b/new.txt")
454 other = ResourcePath(f"{self.root}text.txt", forceDirectory=False)
455 with self.assertRaises(ValueError):
456 other.join("b/new.text")
458 joined = ResourcePath(f"{self.root}hsc/payload/").join(
459 ResourcePath("test.qgraph", forceAbsolute=False)
460 )
461 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph"))
463 qgraph = ResourcePath("test.qgraph") # Absolute URI
464 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph)
465 self.assertEqual(joined, qgraph)
467 with self.assertRaises(ValueError):
468 root.join("dir/", forceDirectory=False)
470 temp = root.join("dir2/", isTemporary=True)
471 with self.assertRaises(RuntimeError):
472 temp.join("test.txt", isTemporary=False)
474 rel = ResourcePath("new.txt", forceAbsolute=False, forceDirectory=False)
475 with self.assertRaises(RuntimeError):
476 root.join(rel, forceDirectory=True)
478 def test_quoting(self) -> None:
479 """Check that quoting works."""
480 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True)
481 subpath = "rootdir/dir1+/file?.txt"
482 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath)))
484 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt")
485 self.assertEqual(child.basename(), "file?.txt")
486 self.assertEqual(child.relativeToPathRoot, subpath)
487 self.assertIn("%", child.path)
488 self.assertEqual(child.unquoted_path, "/" + subpath)
490 def test_ordering(self) -> None:
491 """Check that greater/less comparison operators work."""
492 a = self._make_uri("a.txt")
493 b = self._make_uri("b/")
494 self.assertLess(a, b)
495 self.assertFalse(a < a)
496 self.assertLessEqual(a, b)
497 self.assertLessEqual(a, a)
498 self.assertGreater(b, a)
499 self.assertFalse(b > b)
500 self.assertGreaterEqual(b, a)
501 self.assertGreaterEqual(b, b)
504class GenericReadWriteTestCase(_GenericTestCase):
505 """Test schemes that can read and write using concrete resources."""
507 transfer_modes: tuple[str, ...] = ("copy", "move")
508 testdir: str | None = None
510 def setUp(self) -> None:
511 if self.scheme is None:
512 raise unittest.SkipTest("No scheme defined")
513 self.root = self._make_uri("")
514 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False)
516 if self.scheme == "file":
517 # Use a local tempdir because on macOS the temp dirs use symlinks
518 # so relsymlink gets quite confused.
519 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir), forceDirectory=True)
520 else:
521 # Create random tmp directory relative to the test root.
522 self.tmpdir = self.root_uri.join(
523 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)),
524 forceDirectory=True,
525 )
526 self.tmpdir.mkdir()
528 def tearDown(self) -> None:
529 if self.tmpdir and self.tmpdir.isLocal:
530 removeTestTempDir(self.tmpdir.ospath)
532 def test_file(self) -> None:
533 uri = self.tmpdir.join("test.txt")
534 self.assertFalse(uri.exists(), f"{uri} should not exist")
535 self.assertTrue(uri.path.endswith("test.txt"))
537 content = "abcdefghijklmnopqrstuv\n"
538 uri.write(content.encode())
539 self.assertTrue(uri.exists(), f"{uri} should now exist")
540 self.assertEqual(uri.read().decode(), content)
541 self.assertEqual(uri.size(), len(content.encode()))
543 with self.assertRaises(FileExistsError):
544 uri.write(b"", overwrite=False)
546 # Not all backends can tell if a remove fails so we can not
547 # test that a remove of a non-existent entry is guaranteed to raise.
548 uri.remove()
549 self.assertFalse(uri.exists())
551 # Ideally the test would remove the file again and raise a
552 # FileNotFoundError. This is not reliable for remote resources
553 # and doing an explicit check before trying to remove the resource
554 # just to raise an exception is deemed an unacceptable overhead.
556 with self.assertRaises(FileNotFoundError):
557 uri.read()
559 with self.assertRaises(FileNotFoundError):
560 self.tmpdir.join("file/not/there.txt").size()
562 # Check that creating a URI from a URI returns the same thing
563 uri2 = ResourcePath(uri)
564 self.assertEqual(uri, uri2)
565 self.assertEqual(id(uri), id(uri2))
567 def test_mkdir(self) -> None:
568 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True)
569 newdir.mkdir()
570 self.assertTrue(newdir.exists())
571 self.assertEqual(newdir.size(), 0)
573 newfile = newdir.join("temp.txt")
574 newfile.write(b"Data")
575 self.assertTrue(newfile.exists())
577 file = self.tmpdir.join("file.txt")
578 # Some schemes will realize that the URI is not a file and so
579 # will raise NotADirectoryError. The file scheme is more permissive
580 # and lets you write anything but will raise NotADirectoryError
581 # if a non-directory is already there. We therefore write something
582 # to the file to ensure that we trigger a portable exception.
583 file.write(b"")
584 with self.assertRaises(NotADirectoryError):
585 file.mkdir()
587 # The root should exist.
588 self.root_uri.mkdir()
589 self.assertTrue(self.root_uri.exists())
591 def test_transfer(self) -> None:
592 src = self.tmpdir.join("test.txt")
593 content = "Content is some content\nwith something to say\n\n"
594 src.write(content.encode())
596 can_move = "move" in self.transfer_modes
597 for mode in self.transfer_modes:
598 if mode == "move":
599 continue
601 dest = self.tmpdir.join(f"dest_{mode}.txt")
602 # Ensure that we get some debugging output.
603 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm:
604 dest.transfer_from(src, transfer=mode)
605 self.assertIn("Transferring ", "\n".join(cm.output))
606 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})")
608 new_content = dest.read().decode()
609 self.assertEqual(new_content, content)
611 if mode in ("symlink", "relsymlink"):
612 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink")
614 # If the source and destination are hardlinks of each other
615 # the transfer should work even if overwrite=False.
616 if mode in ("link", "hardlink"):
617 dest.transfer_from(src, transfer=mode)
618 else:
619 with self.assertRaises(
620 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})"
621 ):
622 dest.transfer_from(src, transfer=mode)
624 # Transfer again and overwrite.
625 dest.transfer_from(src, transfer=mode, overwrite=True)
627 dest.remove()
629 b = src.read()
630 self.assertEqual(b.decode(), new_content)
632 nbytes = 10
633 subset = src.read(size=nbytes)
634 self.assertEqual(len(subset), nbytes)
635 self.assertEqual(subset.decode(), content[:nbytes])
637 # Transferring to self should be okay.
638 src.transfer_from(src, "auto")
640 with self.assertRaises(ValueError):
641 src.transfer_from(src, transfer="unknown")
643 # A move transfer is special.
644 if can_move:
645 dest.transfer_from(src, transfer="move")
646 self.assertFalse(src.exists())
647 self.assertTrue(dest.exists())
648 else:
649 src.remove()
651 dest.remove()
652 with self.assertRaises(FileNotFoundError):
653 dest.transfer_from(src, "auto")
655 def test_local_transfer(self) -> None:
656 """Test we can transfer to and from local file."""
657 remote_src = self.tmpdir.join("src.json")
658 remote_src.write(b"42")
659 remote_dest = self.tmpdir.join("dest.json")
661 with ResourcePath.temporary_uri(suffix=".json") as tmp:
662 self.assertTrue(tmp.isLocal)
663 tmp.transfer_from(remote_src, transfer="auto")
664 self.assertEqual(tmp.read(), remote_src.read())
666 remote_dest.transfer_from(tmp, transfer="auto")
667 self.assertEqual(remote_dest.read(), tmp.read())
669 # Temporary (possibly remote) resource.
670 # Transfers between temporary resources.
671 with (
672 ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp,
673 ResourcePath.temporary_uri(suffix=".json") as local_tmp,
674 ):
675 remote_tmp.write(b"42")
676 if not remote_tmp.isLocal:
677 for transfer in ("link", "symlink", "hardlink", "relsymlink"):
678 with self.assertRaises(RuntimeError):
679 # Trying to symlink a remote resource is not going
680 # to work. A hardlink could work but would rely
681 # on the local temp space being on the same
682 # filesystem as the target.
683 local_tmp.transfer_from(remote_tmp, transfer)
684 local_tmp.transfer_from(remote_tmp, "move")
685 self.assertFalse(remote_tmp.exists())
686 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True)
687 self.assertEqual(local_tmp.read(), remote_tmp.read())
689 # Transfer of missing remote.
690 remote_tmp.remove()
691 with self.assertRaises(FileNotFoundError):
692 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True)
694 def test_local(self) -> None:
695 """Check that remote resources can be made local."""
696 src = self.tmpdir.join("test.txt")
697 original_content = "Content is some content\nwith something to say\n\n"
698 src.write(original_content.encode())
700 # Run this twice to ensure use of cache in code coverage
701 # if applicable.
702 for _ in (1, 2):
703 with src.as_local() as local_uri:
704 self.assertTrue(local_uri.isLocal)
705 content = local_uri.read().decode()
706 self.assertEqual(content, original_content)
708 if src.isLocal:
709 self.assertEqual(src, local_uri)
711 with self.assertRaises(IsADirectoryError):
712 with self.root_uri.as_local() as local_uri:
713 pass
715 def test_walk(self) -> None:
716 """Walk a directory hierarchy."""
717 root = self.tmpdir.join("walk/")
719 # Look for a file that is not there
720 file = root.join("config/basic/butler.yaml")
721 found_list = list(ResourcePath.findFileResources([file]))
722 self.assertEqual(found_list[0], file)
724 # First create the files (content is irrelevant).
725 expected_files = {
726 "dir1/a.yaml",
727 "dir1/b.yaml",
728 "dir1/c.json",
729 "dir2/d.json",
730 "dir2/e.yaml",
731 }
732 expected_uris = {root.join(f) for f in expected_files}
733 for uri in expected_uris:
734 uri.write(b"")
735 self.assertTrue(uri.exists())
737 # Look for the files.
738 found = set(ResourcePath.findFileResources([root]))
739 self.assertEqual(found, expected_uris)
741 # Now solely the YAML files.
742 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"}
743 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$"))
744 self.assertEqual(found, expected_yaml)
746 # Now two explicit directories and a file
747 expected = set(expected_yaml)
748 expected.add(file)
750 found = set(
751 ResourcePath.findFileResources(
752 [file, root.join("dir1/"), root.join("dir2/")],
753 file_filter=r".*\.yaml$",
754 )
755 )
756 self.assertEqual(found, expected)
758 # Group by directory -- find everything and compare it with what
759 # we expected to be there in total.
760 found_yaml = set()
761 counter = 0
762 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True):
763 assert not isinstance(uris, ResourcePath) # for mypy.
764 found_uris = set(uris)
765 if found_uris:
766 counter += 1
768 found_yaml.update(found_uris)
770 expected_yaml_2 = expected_yaml
771 expected_yaml_2.add(file)
772 self.assertEqual(found_yaml, expected_yaml)
773 self.assertEqual(counter, 3)
775 # Grouping but check that single files are returned in a single group
776 # at the end
777 file2 = root.join("config/templates/templates-bad.yaml")
778 found_grouped = [
779 list(group)
780 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True)
781 if not isinstance(group, ResourcePath) # For mypy.
782 ]
783 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}")
784 self.assertEqual(list(found_grouped[1]), [file, file2])
786 with self.assertRaises(ValueError):
787 # The list forces the generator to run.
788 list(file.walk())
790 # A directory that does not exist returns nothing.
791 self.assertEqual(list(root.join("dir3/").walk()), [])
793 def test_large_walk(self) -> None:
794 # In some systems pagination is used so ensure that we can handle
795 # large numbers of files. For example S3 limits us to 1000 responses
796 # per listing call.
797 created = set()
798 counter = 1
799 n_dir1 = 1100
800 root = self.tmpdir.join("large_walk", forceDirectory=True)
801 while counter <= n_dir1:
802 new = ResourcePath(root.join(f"file{counter:04d}.txt"))
803 new.write(f"{counter}".encode())
804 created.add(new)
805 counter += 1
806 counter = 1
807 # Put some in a subdirectory to make sure we are looking in a
808 # hierarchy.
809 n_dir2 = 100
810 subdir = root.join("subdir", forceDirectory=True)
811 while counter <= n_dir2:
812 new = ResourcePath(subdir.join(f"file{counter:04d}.txt"))
813 new.write(f"{counter}".encode())
814 created.add(new)
815 counter += 1
817 found = set(ResourcePath.findFileResources([root]))
818 self.assertEqual(len(found), n_dir1 + n_dir2)
819 self.assertEqual(found, created)
821 # Again with grouping.
822 # (mypy gets upset not knowing which of the two options is being
823 # returned so add useless instance check).
824 found_list = [
825 list(group)
826 for group in ResourcePath.findFileResources([root], grouped=True)
827 if not isinstance(group, ResourcePath) # For mypy.
828 ]
829 self.assertEqual(len(found_list), 2)
830 self.assertEqual(len(found_list[0]), n_dir1)
831 self.assertEqual(len(found_list[1]), n_dir2)
833 def test_temporary(self) -> None:
834 prefix = self.tmpdir.join("tmp", forceDirectory=True)
835 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp:
836 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}")
837 self.assertTrue(tmp.isabs(), f"uri: {tmp}")
838 self.assertFalse(tmp.exists(), f"uri: {tmp}")
839 tmp.write(b"abcd")
840 self.assertTrue(tmp.exists(), f"uri: {tmp}")
841 self.assertTrue(tmp.isTemporary)
842 self.assertFalse(tmp.exists(), f"uri: {tmp}")
844 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
845 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp:
846 # Use a specified tmpdir and check it is okay for the file
847 # to not be created.
848 self.assertFalse(tmp.getExtension())
849 self.assertFalse(tmp.exists(), f"uri: {tmp}")
850 self.assertEqual(tmp.scheme, self.scheme)
851 self.assertTrue(tmp.isTemporary)
852 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists")
854 # Fake a directory suffix.
855 with self.assertRaises(NotImplementedError):
856 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp:
857 pass
859 def test_open(self) -> None:
860 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True)
861 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp:
862 _check_open(self, tmp, mode_suffixes=("", "t"))
863 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16")
864 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True)
865 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True)
866 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp:
867 _check_open(self, tmp, mode_suffixes=("b",))
868 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True)
870 with self.assertRaises(IsADirectoryError):
871 with self.root_uri.open():
872 pass
874 def test_mexists(self) -> None:
875 root = self.tmpdir.join("mexists/")
877 # A file that is not there.
878 file = root.join("config/basic/butler.yaml")
880 # Create some files.
881 expected_files = {
882 "dir1/a.yaml",
883 "dir1/b.yaml",
884 "dir2/e.yaml",
885 }
886 expected_uris = {root.join(f) for f in expected_files}
887 for uri in expected_uris:
888 uri.write(b"")
889 self.assertTrue(uri.exists())
890 expected_uris.add(file)
892 multi = ResourcePath.mexists(expected_uris)
894 for uri, is_there in multi.items():
895 if uri == file:
896 self.assertFalse(is_there)
897 else:
898 self.assertTrue(is_there)