Coverage for python/lsst/resources/tests.py: 11%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import unittest 

19import urllib.parse 

20import uuid 

21from typing import Any, Callable, Iterable, Optional, Union 

22 

23from lsst.resources import ResourcePath 

24from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

25 

26TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

27 

28 

29def _check_open( 

30 test_case: Union[_GenericTestCase, unittest.TestCase], 

31 uri: ResourcePath, 

32 *, 

33 mode_suffixes: Iterable[str] = ("", "t", "b"), 

34 **kwargs: Any, 

35) -> None: 

36 """Test an implementation of ButlerURI.open. 

37 

38 Parameters 

39 ---------- 

40 test_case : `unittest.TestCase` 

41 Test case to use for assertions. 

42 uri : `ResourcePath` 

43 URI to use for tests. Must point to a writeable location that is not 

44 yet occupied by a file. On return, the location may point to a file 

45 only if the test fails. 

46 mode_suffixes : `Iterable` of `str` 

47 Suffixes to pass as part of the ``mode`` argument to 

48 `ResourcePath.open`, indicating whether to open as binary or as text; 

49 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

50 **kwargs 

51 Additional keyword arguments to forward to all calls to `open`. 

52 """ 

53 text_content = "wxyz🙂" 

54 bytes_content = uuid.uuid4().bytes 

55 content_by_mode_suffix = { 

56 "": text_content, 

57 "t": text_content, 

58 "b": bytes_content, 

59 } 

60 empty_content_by_mode_suffix = { 

61 "": "", 

62 "t": "", 

63 "b": b"", 

64 } 

65 # To appease mypy 

66 double_content_by_mode_suffix = { 

67 "": text_content + text_content, 

68 "t": text_content + text_content, 

69 "b": bytes_content + bytes_content, 

70 } 

71 for mode_suffix in mode_suffixes: 

72 content = content_by_mode_suffix[mode_suffix] 

73 double_content = double_content_by_mode_suffix[mode_suffix] 

74 # Create file with mode='x', which prohibits overwriting. 

75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

76 write_buffer.write(content) 

77 test_case.assertTrue(uri.exists()) 

78 # Check that opening with 'x' now raises, and does not modify content. 

79 with test_case.assertRaises(FileExistsError): 

80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

81 write_buffer.write("bad") 

82 # Read the file we created and check the contents. 

83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

84 test_case.assertEqual(read_buffer.read(), content) 

85 # Write two copies of the content, overwriting the single copy there. 

86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

87 write_buffer.write(double_content) 

88 # Read again, this time use mode='r+', which reads what is there and 

89 # then lets us write more; we'll use that to reset the file to one 

90 # copy of the content. 

91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

92 test_case.assertEqual(rw_buffer.read(), double_content) 

93 rw_buffer.seek(0) 

94 rw_buffer.truncate() 

95 rw_buffer.write(content) 

96 rw_buffer.seek(0) 

97 test_case.assertEqual(rw_buffer.read(), content) 

98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

99 test_case.assertEqual(read_buffer.read(), content) 

100 # Append some more content to the file; should now have two copies. 

101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

102 append_buffer.write(content) 

103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

104 test_case.assertEqual(read_buffer.read(), double_content) 

105 # Final mode to check is w+, which does read/write but truncates first. 

106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

108 rw_buffer.write(content) 

109 rw_buffer.seek(0) 

110 test_case.assertEqual(rw_buffer.read(), content) 

111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

112 test_case.assertEqual(read_buffer.read(), content) 

113 # Remove file to make room for the next loop of tests with this URI. 

114 uri.remove() 

115 

116 

117class _GenericTestCase: 

118 """Generic base class for test mixin.""" 

119 

120 scheme: Optional[str] = None 

121 netloc: Optional[str] = None 

122 path1 = "test_dir" 

123 path2 = "file.txt" 

124 

125 # Because we use a mixin for tests mypy needs to understand that 

126 # the unittest.TestCase methods exist. 

127 # We do not inherit from unittest.TestCase because that results 

128 # in the tests defined here being run as well as the tests in the 

129 # test file itself. We can make those tests skip but it gives an 

130 # uniformative view of how many tests are running. 

131 assertEqual: Callable 

132 assertNotEqual: Callable 

133 assertIsNone: Callable 

134 assertIn: Callable 

135 assertNotIn: Callable 

136 assertFalse: Callable 

137 assertTrue: Callable 

138 assertRaises: Callable 

139 assertLogs: Callable 

140 

141 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str: 

142 if self.scheme is not None: 

143 if netloc is None: 

144 netloc = self.netloc 

145 if path.startswith("/"): 

146 path = path[1:] 

147 return f"{self.scheme}://{netloc}/{path}" 

148 else: 

149 return path 

150 

151 

152class GenericTestCase(_GenericTestCase): 

153 """Test cases for generic manipulation of a `ResourcePath`""" 

154 

155 def setUp(self) -> None: 

156 if self.scheme is None: 

157 raise unittest.SkipTest("No scheme defined") 

158 self.root = self._make_uri("") 

159 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

160 

161 def test_creation(self) -> None: 

162 self.assertEqual(self.root_uri.scheme, self.scheme) 

163 self.assertEqual(self.root_uri.netloc, self.netloc) 

164 self.assertFalse(self.root_uri.query) 

165 self.assertFalse(self.root_uri.params) 

166 

167 with self.assertRaises(ValueError): 

168 ResourcePath({}) # type: ignore 

169 

170 with self.assertRaises(RuntimeError): 

171 ResourcePath(self.root_uri, isTemporary=True) 

172 

173 file = self.root_uri.join("file.txt") 

174 with self.assertRaises(RuntimeError): 

175 ResourcePath(file, forceDirectory=True) 

176 

177 with self.assertRaises(NotImplementedError): 

178 ResourcePath("unknown://netloc") 

179 

180 replaced = file.replace(fragment="frag") 

181 self.assertEqual(replaced.fragment, "frag") 

182 

183 with self.assertRaises(ValueError): 

184 file.replace(scheme="new") 

185 

186 self.assertNotEqual(replaced, str(replaced)) 

187 self.assertNotEqual(str(replaced), replaced) 

188 

189 def test_extension(self) -> None: 

190 uri = ResourcePath(self._make_uri("dir/test.txt")) 

191 self.assertEqual(uri.updatedExtension(None), uri) 

192 self.assertEqual(uri.updatedExtension(".txt"), uri) 

193 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

194 

195 fits = uri.updatedExtension(".fits.gz") 

196 self.assertEqual(fits.basename(), "test.fits.gz") 

197 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

198 

199 extensionless = self.root_uri.join("no_ext") 

200 self.assertEqual(extensionless.getExtension(), "") 

201 extension = extensionless.updatedExtension(".fits") 

202 self.assertEqual(extension.getExtension(), ".fits") 

203 

204 def test_relative(self) -> None: 

205 """Check that we can get subpaths back from two URIs""" 

206 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

207 self.assertTrue(parent.isdir()) 

208 child = parent.join("dir1/file.txt") 

209 

210 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

211 

212 not_child = ResourcePath("/a/b/dir1/file.txt") 

213 self.assertIsNone(not_child.relative_to(parent)) 

214 self.assertFalse(not_child.isdir()) 

215 

216 not_directory = parent.join("dir1/file2.txt") 

217 self.assertIsNone(child.relative_to(not_directory)) 

218 

219 # Relative URIs 

220 parent = ResourcePath("a/b/", forceAbsolute=False) 

221 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

222 self.assertFalse(child.scheme) 

223 self.assertEqual(child.relative_to(parent), "c/d.txt") 

224 

225 # forceAbsolute=True should work even on an existing ResourcePath 

226 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

227 

228 # Absolute URI and schemeless URI 

229 parent = self.root_uri.join("/a/b/c/") 

230 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

231 

232 # If the child is relative and the parent is absolute we assume 

233 # that the child is a child of the parent unless it uses ".." 

234 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

235 

236 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

237 self.assertIsNone(child.relative_to(parent)) 

238 

239 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

240 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

241 

242 # Test with different netloc 

243 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

244 parent = ResourcePath(self._make_uri("a", netloc="other")) 

245 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

246 

247 # Schemeless absolute child. 

248 # Schemeless absolute URI is constructed using root= parameter. 

249 # For now the root parameter can not be anything other than a file. 

250 if self.scheme == "file": 

251 parent = ResourcePath("/a/b/c", root=self.root_uri) 

252 child = ResourcePath("d/e.txt", root=parent) 

253 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

254 

255 parent = ResourcePath("c/", root="/a/b/") 

256 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

257 

258 # Absolute schemeless child with relative parent will always fail. 

259 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

260 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

261 

262 def test_parents(self) -> None: 

263 """Test of splitting and parent walking.""" 

264 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

265 child_file = parent.join("subdir/file.txt") 

266 self.assertFalse(child_file.isdir()) 

267 child_subdir, file = child_file.split() 

268 self.assertEqual(file, "file.txt") 

269 self.assertTrue(child_subdir.isdir()) 

270 self.assertEqual(child_file.dirname(), child_subdir) 

271 self.assertEqual(child_file.basename(), file) 

272 self.assertEqual(child_file.parent(), child_subdir) 

273 derived_parent = child_subdir.parent() 

274 self.assertEqual(derived_parent, parent) 

275 self.assertTrue(derived_parent.isdir()) 

276 self.assertEqual(child_file.parent().parent(), parent) 

277 

278 def test_escapes(self) -> None: 

279 """Special characters in file paths""" 

280 src = self.root_uri.join("bbb/???/test.txt") 

281 self.assertNotIn("???", src.path) 

282 self.assertIn("???", src.unquoted_path) 

283 

284 file = src.updatedFile("tests??.txt") 

285 self.assertNotIn("??.txt", file.path) 

286 

287 src = src.updatedFile("tests??.txt") 

288 self.assertIn("??.txt", src.unquoted_path) 

289 

290 # File URI and schemeless URI 

291 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

292 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

293 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

294 

295 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

296 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

297 

298 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

299 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

300 

301 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

302 

303 # dir.join() morphs into a file scheme 

304 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

305 new = dir.join("test_j.txt") 

306 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

307 

308 new2name = "###/test??.txt" 

309 new2 = dir.join(new2name) 

310 self.assertIn("???", new2.unquoted_path) 

311 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

312 

313 fdir = dir.abspath() 

314 self.assertNotIn("???", fdir.path) 

315 self.assertIn("???", fdir.unquoted_path) 

316 self.assertEqual(fdir.scheme, self.scheme) 

317 

318 fnew2 = fdir.join(new2name) 

319 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

320 self.assertNotIn("###", fnew2.path) 

321 

322 # Test that children relative to schemeless and file schemes 

323 # still return the same unquoted name 

324 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

325 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

326 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

327 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

328 

329 # Check for double quoting 

330 plus_path = "/a/b/c+d/" 

331 with self.assertLogs(level="WARNING"): 

332 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

333 self.assertEqual(uri.ospath, plus_path) 

334 

335 # Check that # is not escaped for schemeless URIs 

336 hash_path = "/a/b#/c&d#xyz" 

337 hpos = hash_path.rfind("#") 

338 uri = ResourcePath(hash_path) 

339 self.assertEqual(uri.ospath, hash_path[:hpos]) 

340 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

341 

342 def test_hash(self) -> None: 

343 """Test that we can store URIs in sets and as keys.""" 

344 uri1 = self.root_uri 

345 uri2 = uri1.join("test/") 

346 s = {uri1, uri2} 

347 self.assertIn(uri1, s) 

348 

349 d = {uri1: "1", uri2: "2"} 

350 self.assertEqual(d[uri2], "2") 

351 

352 def test_root_uri(self) -> None: 

353 """Test ResourcePath.root_uri().""" 

354 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

355 self.assertEqual(uri.root_uri().geturl(), self.root) 

356 

357 def test_join(self) -> None: 

358 """Test .join method.""" 

359 root_str = self.root 

360 root = self.root_uri 

361 

362 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

363 add_dir = root.join("b/c/d/") 

364 self.assertTrue(add_dir.isdir()) 

365 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

366 

367 up_relative = root.join("../b/c.txt") 

368 self.assertFalse(up_relative.isdir()) 

369 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

370 

371 quote_example = "hsc/payload/b&c.t@x#t" 

372 needs_quote = root.join(quote_example) 

373 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

374 

375 other = ResourcePath(f"{self.root}test.txt") 

376 self.assertEqual(root.join(other), other) 

377 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

378 

379 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

380 ResourcePath("test.qgraph", forceAbsolute=False) 

381 ) 

382 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

383 

384 with self.assertRaises(ValueError): 

385 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph")) 

386 

387 def test_quoting(self) -> None: 

388 """Check that quoting works.""" 

389 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

390 subpath = "rootdir/dir1+/file?.txt" 

391 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

392 

393 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

394 self.assertEqual(child.basename(), "file?.txt") 

395 self.assertEqual(child.relativeToPathRoot, subpath) 

396 self.assertIn("%", child.path) 

397 self.assertEqual(child.unquoted_path, "/" + subpath) 

398 

399 def test_ordering(self) -> None: 

400 """Check that greater/less comparison operators work.""" 

401 a = self._make_uri("a.txt") 

402 b = self._make_uri("b/") 

403 self.assertTrue(a < b) 

404 self.assertFalse(a < a) 

405 self.assertTrue(a <= b) 

406 self.assertTrue(a <= a) 

407 self.assertTrue(b > a) 

408 self.assertFalse(b > b) 

409 self.assertTrue(b >= a) 

410 self.assertTrue(b >= b) 

411 

412 

413class GenericReadWriteTestCase(_GenericTestCase): 

414 """Test schemes that can read and write using concrete resources.""" 

415 

416 transfer_modes = ("copy", "move") 

417 testdir: Optional[str] = None 

418 

419 def setUp(self) -> None: 

420 if self.scheme is None: 

421 raise unittest.SkipTest("No scheme defined") 

422 self.root = self._make_uri("") 

423 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

424 

425 if self.scheme == "file": 

426 # Use a local tempdir because on macOS the temp dirs use symlinks 

427 # so relsymlink gets quite confused. 

428 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

429 else: 

430 # Create tmp directory relative to the test root. 

431 self.tmpdir = self.root_uri.join("TESTING/") 

432 self.tmpdir.mkdir() 

433 

434 def tearDown(self) -> None: 

435 if self.tmpdir: 

436 if self.tmpdir.isLocal: 

437 removeTestTempDir(self.tmpdir.ospath) 

438 

439 def test_file(self) -> None: 

440 uri = self.tmpdir.join("test.txt") 

441 self.assertFalse(uri.exists(), f"{uri} should not exist") 

442 self.assertTrue(uri.path.endswith("test.txt")) 

443 

444 content = "abcdefghijklmnopqrstuv\n" 

445 uri.write(content.encode()) 

446 self.assertTrue(uri.exists(), f"{uri} should now exist") 

447 self.assertEqual(uri.read().decode(), content) 

448 self.assertEqual(uri.size(), len(content.encode())) 

449 

450 with self.assertRaises(FileExistsError): 

451 uri.write(b"", overwrite=False) 

452 

453 # Not all backends can tell if a remove fails so we can not 

454 # test that a remove of a non-existent entry is guaranteed to raise. 

455 uri.remove() 

456 self.assertFalse(uri.exists()) 

457 

458 # Ideally the test would remove the file again and raise a 

459 # FileNotFoundError. This is not reliable for remote resources 

460 # and doing an explicit check before trying to remove the resource 

461 # just to raise an exception is deemed an unacceptable overhead. 

462 

463 with self.assertRaises(FileNotFoundError): 

464 uri.read() 

465 

466 with self.assertRaises(FileNotFoundError): 

467 self.tmpdir.join("file/not/there.txt").size() 

468 

469 # Check that creating a URI from a URI returns the same thing 

470 uri2 = ResourcePath(uri) 

471 self.assertEqual(uri, uri2) 

472 self.assertEqual(id(uri), id(uri2)) 

473 

474 def test_mkdir(self) -> None: 

475 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

476 newdir.mkdir() 

477 self.assertTrue(newdir.exists()) 

478 self.assertEqual(newdir.size(), 0) 

479 

480 newfile = newdir.join("temp.txt") 

481 newfile.write("Data".encode()) 

482 self.assertTrue(newfile.exists()) 

483 

484 file = self.tmpdir.join("file.txt") 

485 # Some schemes will realize that the URI is not a file and so 

486 # will raise NotADirectoryError. The file scheme is more permissive 

487 # and lets you write anything but will raise NotADirectoryError 

488 # if a non-directory is already there. We therefore write something 

489 # to the file to ensure that we trigger a portable exception. 

490 file.write(b"") 

491 with self.assertRaises(NotADirectoryError): 

492 file.mkdir() 

493 

494 # The root should exist. 

495 self.root_uri.mkdir() 

496 self.assertTrue(self.root_uri.exists()) 

497 

498 def test_transfer(self) -> None: 

499 src = self.tmpdir.join("test.txt") 

500 content = "Content is some content\nwith something to say\n\n" 

501 src.write(content.encode()) 

502 

503 can_move = "move" in self.transfer_modes 

504 for mode in self.transfer_modes: 

505 if mode == "move": 

506 continue 

507 

508 dest = self.tmpdir.join(f"dest_{mode}.txt") 

509 # Ensure that we get some debugging output. 

510 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

511 dest.transfer_from(src, transfer=mode) 

512 self.assertIn("Transferring ", "\n".join(cm.output)) 

513 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

514 

515 new_content = dest.read().decode() 

516 self.assertEqual(new_content, content) 

517 

518 if mode in ("symlink", "relsymlink"): 

519 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

520 

521 # If the source and destination are hardlinks of each other 

522 # the transfer should work even if overwrite=False. 

523 if mode in ("link", "hardlink"): 

524 dest.transfer_from(src, transfer=mode) 

525 else: 

526 with self.assertRaises( 

527 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

528 ): 

529 dest.transfer_from(src, transfer=mode) 

530 

531 # Transfer again and overwrite. 

532 dest.transfer_from(src, transfer=mode, overwrite=True) 

533 

534 dest.remove() 

535 

536 b = src.read() 

537 self.assertEqual(b.decode(), new_content) 

538 

539 nbytes = 10 

540 subset = src.read(size=nbytes) 

541 self.assertEqual(len(subset), nbytes) 

542 self.assertEqual(subset.decode(), content[:nbytes]) 

543 

544 # Transferring to self should be okay. 

545 src.transfer_from(src, "auto") 

546 

547 with self.assertRaises(ValueError): 

548 src.transfer_from(src, transfer="unknown") 

549 

550 # A move transfer is special. 

551 if can_move: 

552 dest.transfer_from(src, transfer="move") 

553 self.assertFalse(src.exists()) 

554 self.assertTrue(dest.exists()) 

555 else: 

556 src.remove() 

557 

558 dest.remove() 

559 with self.assertRaises(FileNotFoundError): 

560 dest.transfer_from(src, "auto") 

561 

562 def test_local_transfer(self) -> None: 

563 """Test we can transfer to and from local file.""" 

564 remote_src = self.tmpdir.join("src.json") 

565 remote_src.write(b"42") 

566 remote_dest = self.tmpdir.join("dest.json") 

567 

568 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

569 self.assertTrue(tmp.isLocal) 

570 tmp.transfer_from(remote_src, transfer="auto") 

571 self.assertEqual(tmp.read(), remote_src.read()) 

572 

573 remote_dest.transfer_from(tmp, transfer="auto") 

574 self.assertEqual(remote_dest.read(), tmp.read()) 

575 

576 # Temporary (possibly remote) resource. 

577 # Transfers between temporary resources. 

578 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

579 # Temporary local resource. 

580 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

581 remote_tmp.write(b"42") 

582 if not remote_tmp.isLocal: 

583 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

584 with self.assertRaises(RuntimeError): 

585 # Trying to symlink a remote resource is not going 

586 # to work. A hardlink could work but would rely 

587 # on the local temp space being on the same 

588 # filesystem as the target. 

589 local_tmp.transfer_from(remote_tmp, transfer) 

590 local_tmp.transfer_from(remote_tmp, "move") 

591 self.assertFalse(remote_tmp.exists()) 

592 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

593 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

594 

595 # Transfer of missing remote. 

596 remote_tmp.remove() 

597 with self.assertRaises(FileNotFoundError): 

598 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

599 

600 def test_local(self) -> None: 

601 """Check that remote resources can be made local.""" 

602 src = self.tmpdir.join("test.txt") 

603 original_content = "Content is some content\nwith something to say\n\n" 

604 src.write(original_content.encode()) 

605 

606 # Run this twice to ensure use of cache in code coverage 

607 # if applicable. 

608 for _ in (1, 2): 

609 with src.as_local() as local_uri: 

610 self.assertTrue(local_uri.isLocal) 

611 content = local_uri.read().decode() 

612 self.assertEqual(content, original_content) 

613 

614 if src.isLocal: 

615 self.assertEqual(src, local_uri) 

616 

617 with self.assertRaises(IsADirectoryError): 

618 with self.root_uri.as_local() as local_uri: 

619 pass 

620 

621 def test_walk(self) -> None: 

622 """Walk a directory hierarchy.""" 

623 root = self.tmpdir.join("walk/") 

624 

625 # Look for a file that is not there 

626 file = root.join("config/basic/butler.yaml") 

627 found_list = list(ResourcePath.findFileResources([file])) 

628 self.assertEqual(found_list[0], file) 

629 

630 # First create the files (content is irrelevant). 

631 expected_files = { 

632 "dir1/a.yaml", 

633 "dir1/b.yaml", 

634 "dir1/c.json", 

635 "dir2/d.json", 

636 "dir2/e.yaml", 

637 } 

638 expected_uris = {root.join(f) for f in expected_files} 

639 for uri in expected_uris: 

640 uri.write(b"") 

641 self.assertTrue(uri.exists()) 

642 

643 # Look for the files. 

644 found = set(ResourcePath.findFileResources([root])) 

645 self.assertEqual(found, expected_uris) 

646 

647 # Now solely the YAML files. 

648 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

649 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

650 self.assertEqual(found, expected_yaml) 

651 

652 # Now two explicit directories and a file 

653 expected = set(u for u in expected_yaml) 

654 expected.add(file) 

655 

656 found = set( 

657 ResourcePath.findFileResources( 

658 [file, root.join("dir1/"), root.join("dir2/")], 

659 file_filter=r".*\.yaml$", 

660 ) 

661 ) 

662 self.assertEqual(found, expected) 

663 

664 # Group by directory -- find everything and compare it with what 

665 # we expected to be there in total. 

666 found_yaml = set() 

667 counter = 0 

668 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True): 

669 assert not isinstance(uris, ResourcePath) # for mypy. 

670 found_uris = set(uris) 

671 if found_uris: 

672 counter += 1 

673 

674 found_yaml.update(found_uris) 

675 

676 expected_yaml_2 = expected_yaml 

677 expected_yaml_2.add(file) 

678 self.assertEqual(found_yaml, expected_yaml) 

679 self.assertEqual(counter, 3) 

680 

681 # Grouping but check that single files are returned in a single group 

682 # at the end 

683 file2 = root.join("config/templates/templates-bad.yaml") 

684 found_grouped = [ 

685 [uri for uri in group] 

686 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True) 

687 if not isinstance(group, ResourcePath) # For mypy. 

688 ] 

689 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}") 

690 self.assertEqual(list(found_grouped[1]), [file, file2]) 

691 

692 with self.assertRaises(ValueError): 

693 # The list forces the generator to run. 

694 list(file.walk()) 

695 

696 # A directory that does not exist returns nothing. 

697 self.assertEqual(list(root.join("dir3/").walk()), []) 

698 

699 def test_large_walk(self) -> None: 

700 # In some systems pagination is used so ensure that we can handle 

701 # large numbers of files. For example S3 limits us to 1000 responses 

702 # per listing call. 

703 created = set() 

704 counter = 1 

705 n_dir1 = 1100 

706 root = self.tmpdir.join("large_walk", forceDirectory=True) 

707 while counter <= n_dir1: 

708 new = ResourcePath(root.join(f"file{counter:04d}.txt")) 

709 new.write(f"{counter}".encode()) 

710 created.add(new) 

711 counter += 1 

712 counter = 1 

713 # Put some in a subdirectory to make sure we are looking in a 

714 # hierarchy. 

715 n_dir2 = 100 

716 subdir = root.join("subdir", forceDirectory=True) 

717 while counter <= n_dir2: 

718 new = ResourcePath(subdir.join(f"file{counter:04d}.txt")) 

719 new.write(f"{counter}".encode()) 

720 created.add(new) 

721 counter += 1 

722 

723 found = set(ResourcePath.findFileResources([root])) 

724 self.assertEqual(len(found), n_dir1 + n_dir2)