Coverage for python/lsst/resources/tests.py: 9%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import random 

19import string 

20import unittest 

21import urllib.parse 

22import uuid 

23from typing import Any, Callable, Iterable, Optional, Union 

24 

25from lsst.resources import ResourcePath 

26from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

27 

28TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

29 

30 

31def _check_open( 

32 test_case: Union[_GenericTestCase, unittest.TestCase], 

33 uri: ResourcePath, 

34 *, 

35 mode_suffixes: Iterable[str] = ("", "t", "b"), 

36 **kwargs: Any, 

37) -> None: 

38 """Test an implementation of ButlerURI.open. 

39 

40 Parameters 

41 ---------- 

42 test_case : `unittest.TestCase` 

43 Test case to use for assertions. 

44 uri : `ResourcePath` 

45 URI to use for tests. Must point to a writeable location that is not 

46 yet occupied by a file. On return, the location may point to a file 

47 only if the test fails. 

48 mode_suffixes : `Iterable` of `str` 

49 Suffixes to pass as part of the ``mode`` argument to 

50 `ResourcePath.open`, indicating whether to open as binary or as text; 

51 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

52 **kwargs 

53 Additional keyword arguments to forward to all calls to `open`. 

54 """ 

55 text_content = "wxyz🙂" 

56 bytes_content = uuid.uuid4().bytes 

57 content_by_mode_suffix = { 

58 "": text_content, 

59 "t": text_content, 

60 "b": bytes_content, 

61 } 

62 empty_content_by_mode_suffix = { 

63 "": "", 

64 "t": "", 

65 "b": b"", 

66 } 

67 # To appease mypy 

68 double_content_by_mode_suffix = { 

69 "": text_content + text_content, 

70 "t": text_content + text_content, 

71 "b": bytes_content + bytes_content, 

72 } 

73 for mode_suffix in mode_suffixes: 

74 content = content_by_mode_suffix[mode_suffix] 

75 double_content = double_content_by_mode_suffix[mode_suffix] 

76 # Create file with mode='x', which prohibits overwriting. 

77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

78 write_buffer.write(content) 

79 test_case.assertTrue(uri.exists()) 

80 # Check that opening with 'x' now raises, and does not modify content. 

81 with test_case.assertRaises(FileExistsError): 

82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

83 write_buffer.write("bad") 

84 # Read the file we created and check the contents. 

85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

86 test_case.assertEqual(read_buffer.read(), content) 

87 # Write two copies of the content, overwriting the single copy there. 

88 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

89 write_buffer.write(double_content) 

90 # Read again, this time use mode='r+', which reads what is there and 

91 # then lets us write more; we'll use that to reset the file to one 

92 # copy of the content. 

93 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

94 test_case.assertEqual(rw_buffer.read(), double_content) 

95 rw_buffer.seek(0) 

96 rw_buffer.truncate() 

97 rw_buffer.write(content) 

98 rw_buffer.seek(0) 

99 test_case.assertEqual(rw_buffer.read(), content) 

100 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

101 test_case.assertEqual(read_buffer.read(), content) 

102 # Append some more content to the file; should now have two copies. 

103 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

104 append_buffer.write(content) 

105 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

106 test_case.assertEqual(read_buffer.read(), double_content) 

107 # Final mode to check is w+, which does read/write but truncates first. 

108 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

109 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

110 rw_buffer.write(content) 

111 rw_buffer.seek(0) 

112 test_case.assertEqual(rw_buffer.read(), content) 

113 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

114 test_case.assertEqual(read_buffer.read(), content) 

115 # Remove file to make room for the next loop of tests with this URI. 

116 uri.remove() 

117 

118 

119class _GenericTestCase: 

120 """Generic base class for test mixin.""" 

121 

122 scheme: Optional[str] = None 

123 netloc: Optional[str] = None 

124 base_path: Optional[str] = None 

125 path1 = "test_dir" 

126 path2 = "file.txt" 

127 

128 # Because we use a mixin for tests mypy needs to understand that 

129 # the unittest.TestCase methods exist. 

130 # We do not inherit from unittest.TestCase because that results 

131 # in the tests defined here being run as well as the tests in the 

132 # test file itself. We can make those tests skip but it gives an 

133 # uniformative view of how many tests are running. 

134 assertEqual: Callable 

135 assertNotEqual: Callable 

136 assertIsNone: Callable 

137 assertIn: Callable 

138 assertNotIn: Callable 

139 assertFalse: Callable 

140 assertTrue: Callable 

141 assertRaises: Callable 

142 assertLogs: Callable 

143 

144 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str: 

145 if self.scheme is not None: 

146 if netloc is None: 

147 netloc = self.netloc 

148 if path.startswith("/"): 

149 path = path[1:] 

150 if self.base_path is not None: 

151 path = f"{self.base_path}{path}".lstrip("/") 

152 

153 return f"{self.scheme}://{netloc}/{path}" 

154 else: 

155 return path 

156 

157 

158class GenericTestCase(_GenericTestCase): 

159 """Test cases for generic manipulation of a `ResourcePath`""" 

160 

161 def setUp(self) -> None: 

162 if self.scheme is None: 

163 raise unittest.SkipTest("No scheme defined") 

164 self.root = self._make_uri("") 

165 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

166 

167 def test_creation(self) -> None: 

168 self.assertEqual(self.root_uri.scheme, self.scheme) 

169 self.assertEqual(self.root_uri.netloc, self.netloc) 

170 self.assertFalse(self.root_uri.query) 

171 self.assertFalse(self.root_uri.params) 

172 

173 with self.assertRaises(ValueError): 

174 ResourcePath({}) # type: ignore 

175 

176 with self.assertRaises(RuntimeError): 

177 ResourcePath(self.root_uri, isTemporary=True) 

178 

179 file = self.root_uri.join("file.txt") 

180 with self.assertRaises(RuntimeError): 

181 ResourcePath(file, forceDirectory=True) 

182 

183 with self.assertRaises(NotImplementedError): 

184 ResourcePath("unknown://netloc") 

185 

186 replaced = file.replace(fragment="frag") 

187 self.assertEqual(replaced.fragment, "frag") 

188 

189 with self.assertRaises(ValueError): 

190 file.replace(scheme="new") 

191 

192 self.assertNotEqual(replaced, str(replaced)) 

193 self.assertNotEqual(str(replaced), replaced) 

194 

195 def test_extension(self) -> None: 

196 uri = ResourcePath(self._make_uri("dir/test.txt")) 

197 self.assertEqual(uri.updatedExtension(None), uri) 

198 self.assertEqual(uri.updatedExtension(".txt"), uri) 

199 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

200 

201 fits = uri.updatedExtension(".fits.gz") 

202 self.assertEqual(fits.basename(), "test.fits.gz") 

203 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

204 

205 extensionless = self.root_uri.join("no_ext") 

206 self.assertEqual(extensionless.getExtension(), "") 

207 extension = extensionless.updatedExtension(".fits") 

208 self.assertEqual(extension.getExtension(), ".fits") 

209 

210 def test_relative(self) -> None: 

211 """Check that we can get subpaths back from two URIs""" 

212 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

213 self.assertTrue(parent.isdir()) 

214 child = parent.join("dir1/file.txt") 

215 

216 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

217 

218 not_child = ResourcePath("/a/b/dir1/file.txt") 

219 self.assertIsNone(not_child.relative_to(parent)) 

220 self.assertFalse(not_child.isdir()) 

221 

222 not_directory = parent.join("dir1/file2.txt") 

223 self.assertIsNone(child.relative_to(not_directory)) 

224 

225 # Relative URIs 

226 parent = ResourcePath("a/b/", forceAbsolute=False) 

227 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

228 self.assertFalse(child.scheme) 

229 self.assertEqual(child.relative_to(parent), "c/d.txt") 

230 

231 # forceAbsolute=True should work even on an existing ResourcePath 

232 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

233 

234 # Absolute URI and schemeless URI 

235 parent = self.root_uri.join("/a/b/c/") 

236 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

237 

238 # If the child is relative and the parent is absolute we assume 

239 # that the child is a child of the parent unless it uses ".." 

240 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

241 

242 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

243 self.assertIsNone(child.relative_to(parent)) 

244 

245 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

246 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

247 

248 # Test with different netloc 

249 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

250 parent = ResourcePath(self._make_uri("a", netloc="other")) 

251 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

252 

253 # Schemeless absolute child. 

254 # Schemeless absolute URI is constructed using root= parameter. 

255 # For now the root parameter can not be anything other than a file. 

256 if self.scheme == "file": 

257 parent = ResourcePath("/a/b/c", root=self.root_uri) 

258 child = ResourcePath("d/e.txt", root=parent) 

259 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

260 

261 parent = ResourcePath("c/", root="/a/b/") 

262 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

263 

264 # Absolute schemeless child with relative parent will always fail. 

265 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

266 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

267 

268 def test_parents(self) -> None: 

269 """Test of splitting and parent walking.""" 

270 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

271 child_file = parent.join("subdir/file.txt") 

272 self.assertFalse(child_file.isdir()) 

273 child_subdir, file = child_file.split() 

274 self.assertEqual(file, "file.txt") 

275 self.assertTrue(child_subdir.isdir()) 

276 self.assertEqual(child_file.dirname(), child_subdir) 

277 self.assertEqual(child_file.basename(), file) 

278 self.assertEqual(child_file.parent(), child_subdir) 

279 derived_parent = child_subdir.parent() 

280 self.assertEqual(derived_parent, parent) 

281 self.assertTrue(derived_parent.isdir()) 

282 self.assertEqual(child_file.parent().parent(), parent) 

283 

284 def test_escapes(self) -> None: 

285 """Special characters in file paths""" 

286 src = self.root_uri.join("bbb/???/test.txt") 

287 self.assertNotIn("???", src.path) 

288 self.assertIn("???", src.unquoted_path) 

289 

290 file = src.updatedFile("tests??.txt") 

291 self.assertNotIn("??.txt", file.path) 

292 

293 src = src.updatedFile("tests??.txt") 

294 self.assertIn("??.txt", src.unquoted_path) 

295 

296 # File URI and schemeless URI 

297 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

298 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

299 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

300 

301 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

302 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

303 

304 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

305 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

306 

307 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

308 

309 # dir.join() morphs into a file scheme 

310 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

311 new = dir.join("test_j.txt") 

312 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

313 

314 new2name = "###/test??.txt" 

315 new2 = dir.join(new2name) 

316 self.assertIn("???", new2.unquoted_path) 

317 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

318 

319 fdir = dir.abspath() 

320 self.assertNotIn("???", fdir.path) 

321 self.assertIn("???", fdir.unquoted_path) 

322 self.assertEqual(fdir.scheme, self.scheme) 

323 

324 fnew2 = fdir.join(new2name) 

325 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

326 self.assertNotIn("###", fnew2.path) 

327 

328 # Test that children relative to schemeless and file schemes 

329 # still return the same unquoted name 

330 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

331 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

332 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

333 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

334 

335 # Check for double quoting 

336 plus_path = "/a/b/c+d/" 

337 with self.assertLogs(level="WARNING"): 

338 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

339 self.assertEqual(uri.ospath, plus_path) 

340 

341 # Check that # is not escaped for schemeless URIs 

342 hash_path = "/a/b#/c&d#xyz" 

343 hpos = hash_path.rfind("#") 

344 uri = ResourcePath(hash_path) 

345 self.assertEqual(uri.ospath, hash_path[:hpos]) 

346 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

347 

348 def test_hash(self) -> None: 

349 """Test that we can store URIs in sets and as keys.""" 

350 uri1 = self.root_uri 

351 uri2 = uri1.join("test/") 

352 s = {uri1, uri2} 

353 self.assertIn(uri1, s) 

354 

355 d = {uri1: "1", uri2: "2"} 

356 self.assertEqual(d[uri2], "2") 

357 

358 def test_root_uri(self) -> None: 

359 """Test ResourcePath.root_uri().""" 

360 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

361 self.assertEqual(uri.root_uri().geturl(), self.root) 

362 

363 def test_join(self) -> None: 

364 """Test .join method.""" 

365 root_str = self.root 

366 root = self.root_uri 

367 

368 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

369 add_dir = root.join("b/c/d/") 

370 self.assertTrue(add_dir.isdir()) 

371 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

372 

373 up_relative = root.join("../b/c.txt") 

374 self.assertFalse(up_relative.isdir()) 

375 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

376 

377 quote_example = "hsc/payload/b&c.t@x#t" 

378 needs_quote = root.join(quote_example) 

379 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

380 

381 other = ResourcePath(f"{self.root}test.txt") 

382 self.assertEqual(root.join(other), other) 

383 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

384 

385 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

386 ResourcePath("test.qgraph", forceAbsolute=False) 

387 ) 

388 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

389 

390 with self.assertRaises(ValueError): 

391 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph")) 

392 

393 def test_quoting(self) -> None: 

394 """Check that quoting works.""" 

395 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

396 subpath = "rootdir/dir1+/file?.txt" 

397 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

398 

399 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

400 self.assertEqual(child.basename(), "file?.txt") 

401 self.assertEqual(child.relativeToPathRoot, subpath) 

402 self.assertIn("%", child.path) 

403 self.assertEqual(child.unquoted_path, "/" + subpath) 

404 

405 def test_ordering(self) -> None: 

406 """Check that greater/less comparison operators work.""" 

407 a = self._make_uri("a.txt") 

408 b = self._make_uri("b/") 

409 self.assertTrue(a < b) 

410 self.assertFalse(a < a) 

411 self.assertTrue(a <= b) 

412 self.assertTrue(a <= a) 

413 self.assertTrue(b > a) 

414 self.assertFalse(b > b) 

415 self.assertTrue(b >= a) 

416 self.assertTrue(b >= b) 

417 

418 

419class GenericReadWriteTestCase(_GenericTestCase): 

420 """Test schemes that can read and write using concrete resources.""" 

421 

422 transfer_modes = ("copy", "move") 

423 testdir: Optional[str] = None 

424 

425 def setUp(self) -> None: 

426 if self.scheme is None: 

427 raise unittest.SkipTest("No scheme defined") 

428 self.root = self._make_uri("") 

429 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

430 

431 if self.scheme == "file": 

432 # Use a local tempdir because on macOS the temp dirs use symlinks 

433 # so relsymlink gets quite confused. 

434 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

435 else: 

436 # Create random tmp directory relative to the test root. 

437 self.tmpdir = self.root_uri.join( 

438 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)), 

439 forceDirectory=True, 

440 ) 

441 self.tmpdir.mkdir() 

442 

443 def tearDown(self) -> None: 

444 if self.tmpdir: 

445 if self.tmpdir.isLocal: 

446 removeTestTempDir(self.tmpdir.ospath) 

447 

448 def test_file(self) -> None: 

449 uri = self.tmpdir.join("test.txt") 

450 self.assertFalse(uri.exists(), f"{uri} should not exist") 

451 self.assertTrue(uri.path.endswith("test.txt")) 

452 

453 content = "abcdefghijklmnopqrstuv\n" 

454 uri.write(content.encode()) 

455 self.assertTrue(uri.exists(), f"{uri} should now exist") 

456 self.assertEqual(uri.read().decode(), content) 

457 self.assertEqual(uri.size(), len(content.encode())) 

458 

459 with self.assertRaises(FileExistsError): 

460 uri.write(b"", overwrite=False) 

461 

462 # Not all backends can tell if a remove fails so we can not 

463 # test that a remove of a non-existent entry is guaranteed to raise. 

464 uri.remove() 

465 self.assertFalse(uri.exists()) 

466 

467 # Ideally the test would remove the file again and raise a 

468 # FileNotFoundError. This is not reliable for remote resources 

469 # and doing an explicit check before trying to remove the resource 

470 # just to raise an exception is deemed an unacceptable overhead. 

471 

472 with self.assertRaises(FileNotFoundError): 

473 uri.read() 

474 

475 with self.assertRaises(FileNotFoundError): 

476 self.tmpdir.join("file/not/there.txt").size() 

477 

478 # Check that creating a URI from a URI returns the same thing 

479 uri2 = ResourcePath(uri) 

480 self.assertEqual(uri, uri2) 

481 self.assertEqual(id(uri), id(uri2)) 

482 

483 def test_mkdir(self) -> None: 

484 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

485 newdir.mkdir() 

486 self.assertTrue(newdir.exists()) 

487 self.assertEqual(newdir.size(), 0) 

488 

489 newfile = newdir.join("temp.txt") 

490 newfile.write("Data".encode()) 

491 self.assertTrue(newfile.exists()) 

492 

493 file = self.tmpdir.join("file.txt") 

494 # Some schemes will realize that the URI is not a file and so 

495 # will raise NotADirectoryError. The file scheme is more permissive 

496 # and lets you write anything but will raise NotADirectoryError 

497 # if a non-directory is already there. We therefore write something 

498 # to the file to ensure that we trigger a portable exception. 

499 file.write(b"") 

500 with self.assertRaises(NotADirectoryError): 

501 file.mkdir() 

502 

503 # The root should exist. 

504 self.root_uri.mkdir() 

505 self.assertTrue(self.root_uri.exists()) 

506 

507 def test_transfer(self) -> None: 

508 src = self.tmpdir.join("test.txt") 

509 content = "Content is some content\nwith something to say\n\n" 

510 src.write(content.encode()) 

511 

512 can_move = "move" in self.transfer_modes 

513 for mode in self.transfer_modes: 

514 if mode == "move": 

515 continue 

516 

517 dest = self.tmpdir.join(f"dest_{mode}.txt") 

518 # Ensure that we get some debugging output. 

519 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

520 dest.transfer_from(src, transfer=mode) 

521 self.assertIn("Transferring ", "\n".join(cm.output)) 

522 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

523 

524 new_content = dest.read().decode() 

525 self.assertEqual(new_content, content) 

526 

527 if mode in ("symlink", "relsymlink"): 

528 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

529 

530 # If the source and destination are hardlinks of each other 

531 # the transfer should work even if overwrite=False. 

532 if mode in ("link", "hardlink"): 

533 dest.transfer_from(src, transfer=mode) 

534 else: 

535 with self.assertRaises( 

536 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

537 ): 

538 dest.transfer_from(src, transfer=mode) 

539 

540 # Transfer again and overwrite. 

541 dest.transfer_from(src, transfer=mode, overwrite=True) 

542 

543 dest.remove() 

544 

545 b = src.read() 

546 self.assertEqual(b.decode(), new_content) 

547 

548 nbytes = 10 

549 subset = src.read(size=nbytes) 

550 self.assertEqual(len(subset), nbytes) 

551 self.assertEqual(subset.decode(), content[:nbytes]) 

552 

553 # Transferring to self should be okay. 

554 src.transfer_from(src, "auto") 

555 

556 with self.assertRaises(ValueError): 

557 src.transfer_from(src, transfer="unknown") 

558 

559 # A move transfer is special. 

560 if can_move: 

561 dest.transfer_from(src, transfer="move") 

562 self.assertFalse(src.exists()) 

563 self.assertTrue(dest.exists()) 

564 else: 

565 src.remove() 

566 

567 dest.remove() 

568 with self.assertRaises(FileNotFoundError): 

569 dest.transfer_from(src, "auto") 

570 

571 def test_local_transfer(self) -> None: 

572 """Test we can transfer to and from local file.""" 

573 remote_src = self.tmpdir.join("src.json") 

574 remote_src.write(b"42") 

575 remote_dest = self.tmpdir.join("dest.json") 

576 

577 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

578 self.assertTrue(tmp.isLocal) 

579 tmp.transfer_from(remote_src, transfer="auto") 

580 self.assertEqual(tmp.read(), remote_src.read()) 

581 

582 remote_dest.transfer_from(tmp, transfer="auto") 

583 self.assertEqual(remote_dest.read(), tmp.read()) 

584 

585 # Temporary (possibly remote) resource. 

586 # Transfers between temporary resources. 

587 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

588 # Temporary local resource. 

589 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

590 remote_tmp.write(b"42") 

591 if not remote_tmp.isLocal: 

592 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

593 with self.assertRaises(RuntimeError): 

594 # Trying to symlink a remote resource is not going 

595 # to work. A hardlink could work but would rely 

596 # on the local temp space being on the same 

597 # filesystem as the target. 

598 local_tmp.transfer_from(remote_tmp, transfer) 

599 local_tmp.transfer_from(remote_tmp, "move") 

600 self.assertFalse(remote_tmp.exists()) 

601 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

602 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

603 

604 # Transfer of missing remote. 

605 remote_tmp.remove() 

606 with self.assertRaises(FileNotFoundError): 

607 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

608 

609 def test_local(self) -> None: 

610 """Check that remote resources can be made local.""" 

611 src = self.tmpdir.join("test.txt") 

612 original_content = "Content is some content\nwith something to say\n\n" 

613 src.write(original_content.encode()) 

614 

615 # Run this twice to ensure use of cache in code coverage 

616 # if applicable. 

617 for _ in (1, 2): 

618 with src.as_local() as local_uri: 

619 self.assertTrue(local_uri.isLocal) 

620 content = local_uri.read().decode() 

621 self.assertEqual(content, original_content) 

622 

623 if src.isLocal: 

624 self.assertEqual(src, local_uri) 

625 

626 with self.assertRaises(IsADirectoryError): 

627 with self.root_uri.as_local() as local_uri: 

628 pass 

629 

630 def test_walk(self) -> None: 

631 """Walk a directory hierarchy.""" 

632 root = self.tmpdir.join("walk/") 

633 

634 # Look for a file that is not there 

635 file = root.join("config/basic/butler.yaml") 

636 found_list = list(ResourcePath.findFileResources([file])) 

637 self.assertEqual(found_list[0], file) 

638 

639 # First create the files (content is irrelevant). 

640 expected_files = { 

641 "dir1/a.yaml", 

642 "dir1/b.yaml", 

643 "dir1/c.json", 

644 "dir2/d.json", 

645 "dir2/e.yaml", 

646 } 

647 expected_uris = {root.join(f) for f in expected_files} 

648 for uri in expected_uris: 

649 uri.write(b"") 

650 self.assertTrue(uri.exists()) 

651 

652 # Look for the files. 

653 found = set(ResourcePath.findFileResources([root])) 

654 self.assertEqual(found, expected_uris) 

655 

656 # Now solely the YAML files. 

657 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

658 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

659 self.assertEqual(found, expected_yaml) 

660 

661 # Now two explicit directories and a file 

662 expected = set(u for u in expected_yaml) 

663 expected.add(file) 

664 

665 found = set( 

666 ResourcePath.findFileResources( 

667 [file, root.join("dir1/"), root.join("dir2/")], 

668 file_filter=r".*\.yaml$", 

669 ) 

670 ) 

671 self.assertEqual(found, expected) 

672 

673 # Group by directory -- find everything and compare it with what 

674 # we expected to be there in total. 

675 found_yaml = set() 

676 counter = 0 

677 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True): 

678 assert not isinstance(uris, ResourcePath) # for mypy. 

679 found_uris = set(uris) 

680 if found_uris: 

681 counter += 1 

682 

683 found_yaml.update(found_uris) 

684 

685 expected_yaml_2 = expected_yaml 

686 expected_yaml_2.add(file) 

687 self.assertEqual(found_yaml, expected_yaml) 

688 self.assertEqual(counter, 3) 

689 

690 # Grouping but check that single files are returned in a single group 

691 # at the end 

692 file2 = root.join("config/templates/templates-bad.yaml") 

693 found_grouped = [ 

694 [uri for uri in group] 

695 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True) 

696 if not isinstance(group, ResourcePath) # For mypy. 

697 ] 

698 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}") 

699 self.assertEqual(list(found_grouped[1]), [file, file2]) 

700 

701 with self.assertRaises(ValueError): 

702 # The list forces the generator to run. 

703 list(file.walk()) 

704 

705 # A directory that does not exist returns nothing. 

706 self.assertEqual(list(root.join("dir3/").walk()), []) 

707 

708 def test_large_walk(self) -> None: 

709 # In some systems pagination is used so ensure that we can handle 

710 # large numbers of files. For example S3 limits us to 1000 responses 

711 # per listing call. 

712 created = set() 

713 counter = 1 

714 n_dir1 = 1100 

715 root = self.tmpdir.join("large_walk", forceDirectory=True) 

716 while counter <= n_dir1: 

717 new = ResourcePath(root.join(f"file{counter:04d}.txt")) 

718 new.write(f"{counter}".encode()) 

719 created.add(new) 

720 counter += 1 

721 counter = 1 

722 # Put some in a subdirectory to make sure we are looking in a 

723 # hierarchy. 

724 n_dir2 = 100 

725 subdir = root.join("subdir", forceDirectory=True)