Coverage for python/lsst/resources/tests.py: 9%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import random 

19import string 

20import unittest 

21import urllib.parse 

22import uuid 

23from typing import Any, Callable, Iterable, Optional, Union 

24 

25from lsst.resources import ResourcePath 

26from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

27 

28TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

29 

30 

31def _check_open( 

32 test_case: Union[_GenericTestCase, unittest.TestCase], 

33 uri: ResourcePath, 

34 *, 

35 mode_suffixes: Iterable[str] = ("", "t", "b"), 

36 **kwargs: Any, 

37) -> None: 

38 """Test an implementation of ButlerURI.open. 

39 

40 Parameters 

41 ---------- 

42 test_case : `unittest.TestCase` 

43 Test case to use for assertions. 

44 uri : `ResourcePath` 

45 URI to use for tests. Must point to a writeable location that is not 

46 yet occupied by a file. On return, the location may point to a file 

47 only if the test fails. 

48 mode_suffixes : `Iterable` of `str` 

49 Suffixes to pass as part of the ``mode`` argument to 

50 `ResourcePath.open`, indicating whether to open as binary or as text; 

51 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

52 **kwargs 

53 Additional keyword arguments to forward to all calls to `open`. 

54 """ 

55 text_content = "wxyz🙂" 

56 bytes_content = uuid.uuid4().bytes 

57 content_by_mode_suffix = { 

58 "": text_content, 

59 "t": text_content, 

60 "b": bytes_content, 

61 } 

62 empty_content_by_mode_suffix = { 

63 "": "", 

64 "t": "", 

65 "b": b"", 

66 } 

67 # To appease mypy 

68 double_content_by_mode_suffix = { 

69 "": text_content + text_content, 

70 "t": text_content + text_content, 

71 "b": bytes_content + bytes_content, 

72 } 

73 for mode_suffix in mode_suffixes: 

74 content = content_by_mode_suffix[mode_suffix] 

75 double_content = double_content_by_mode_suffix[mode_suffix] 

76 # Create file with mode='x', which prohibits overwriting. 

77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

78 write_buffer.write(content) 

79 test_case.assertTrue(uri.exists()) 

80 # Check that opening with 'x' now raises, and does not modify content. 

81 with test_case.assertRaises(FileExistsError): 

82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

83 write_buffer.write("bad") 

84 # Read the file we created and check the contents. 

85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

86 test_case.assertEqual(read_buffer.read(), content) 

87 # Write two copies of the content, overwriting the single copy there. 

88 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

89 write_buffer.write(double_content) 

90 # Read again, this time use mode='r+', which reads what is there and 

91 # then lets us write more; we'll use that to reset the file to one 

92 # copy of the content. 

93 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

94 test_case.assertEqual(rw_buffer.read(), double_content) 

95 rw_buffer.seek(0) 

96 rw_buffer.truncate() 

97 rw_buffer.write(content) 

98 rw_buffer.seek(0) 

99 test_case.assertEqual(rw_buffer.read(), content) 

100 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

101 test_case.assertEqual(read_buffer.read(), content) 

102 # Append some more content to the file; should now have two copies. 

103 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

104 append_buffer.write(content) 

105 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

106 test_case.assertEqual(read_buffer.read(), double_content) 

107 # Final mode to check is w+, which does read/write but truncates first. 

108 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

109 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

110 rw_buffer.write(content) 

111 rw_buffer.seek(0) 

112 test_case.assertEqual(rw_buffer.read(), content) 

113 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

114 test_case.assertEqual(read_buffer.read(), content) 

115 # Remove file to make room for the next loop of tests with this URI. 

116 uri.remove() 

117 

118 

119class _GenericTestCase: 

120 """Generic base class for test mixin.""" 

121 

122 scheme: Optional[str] = None 

123 netloc: Optional[str] = None 

124 base_path: Optional[str] = None 

125 path1 = "test_dir" 

126 path2 = "file.txt" 

127 

128 # Because we use a mixin for tests mypy needs to understand that 

129 # the unittest.TestCase methods exist. 

130 # We do not inherit from unittest.TestCase because that results 

131 # in the tests defined here being run as well as the tests in the 

132 # test file itself. We can make those tests skip but it gives an 

133 # uniformative view of how many tests are running. 

134 assertEqual: Callable 

135 assertNotEqual: Callable 

136 assertIsNone: Callable 

137 assertIn: Callable 

138 assertNotIn: Callable 

139 assertFalse: Callable 

140 assertTrue: Callable 

141 assertRaises: Callable 

142 assertLogs: Callable 

143 

144 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str: 

145 if self.scheme is not None: 

146 if netloc is None: 

147 netloc = self.netloc 

148 if path.startswith("/"): 

149 path = path[1:] 

150 if self.base_path is not None: 

151 path = f"{self.base_path}/{path}".lstrip("/") 

152 

153 return f"{self.scheme}://{netloc}/{path}" 

154 else: 

155 return path 

156 

157 

158class GenericTestCase(_GenericTestCase): 

159 """Test cases for generic manipulation of a `ResourcePath`""" 

160 

161 def setUp(self) -> None: 

162 if self.scheme is None: 

163 raise unittest.SkipTest("No scheme defined") 

164 self.root = self._make_uri("") 

165 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

166 

167 def test_creation(self) -> None: 

168 self.assertEqual(self.root_uri.scheme, self.scheme) 

169 self.assertEqual(self.root_uri.netloc, self.netloc) 

170 self.assertFalse(self.root_uri.query) 

171 self.assertFalse(self.root_uri.params) 

172 

173 with self.assertRaises(ValueError): 

174 ResourcePath({}) # type: ignore 

175 

176 with self.assertRaises(RuntimeError): 

177 ResourcePath(self.root_uri, isTemporary=True) 

178 

179 file = self.root_uri.join("file.txt") 

180 with self.assertRaises(RuntimeError): 

181 ResourcePath(file, forceDirectory=True) 

182 

183 with self.assertRaises(NotImplementedError): 

184 ResourcePath("unknown://netloc") 

185 

186 replaced = file.replace(fragment="frag") 

187 self.assertEqual(replaced.fragment, "frag") 

188 

189 with self.assertRaises(ValueError): 

190 file.replace(scheme="new") 

191 

192 self.assertNotEqual(replaced, str(replaced)) 

193 self.assertNotEqual(str(replaced), replaced) 

194 

195 def test_extension(self) -> None: 

196 uri = ResourcePath(self._make_uri("dir/test.txt")) 

197 self.assertEqual(uri.updatedExtension(None), uri) 

198 self.assertEqual(uri.updatedExtension(".txt"), uri) 

199 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

200 

201 fits = uri.updatedExtension(".fits.gz") 

202 self.assertEqual(fits.basename(), "test.fits.gz") 

203 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

204 

205 extensionless = self.root_uri.join("no_ext") 

206 self.assertEqual(extensionless.getExtension(), "") 

207 extension = extensionless.updatedExtension(".fits") 

208 self.assertEqual(extension.getExtension(), ".fits") 

209 

210 def test_relative(self) -> None: 

211 """Check that we can get subpaths back from two URIs""" 

212 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

213 self.assertTrue(parent.isdir()) 

214 child = parent.join("dir1/file.txt") 

215 

216 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

217 

218 not_child = ResourcePath("/a/b/dir1/file.txt") 

219 self.assertIsNone(not_child.relative_to(parent)) 

220 self.assertFalse(not_child.isdir()) 

221 

222 not_directory = parent.join("dir1/file2.txt") 

223 self.assertIsNone(child.relative_to(not_directory)) 

224 

225 # Relative URIs 

226 parent = ResourcePath("a/b/", forceAbsolute=False) 

227 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

228 self.assertFalse(child.scheme) 

229 self.assertEqual(child.relative_to(parent), "c/d.txt") 

230 

231 # forceAbsolute=True should work even on an existing ResourcePath 

232 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

233 

234 # Absolute URI and schemeless URI 

235 parent = self.root_uri.join("/a/b/c/") 

236 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

237 

238 # If the child is relative and the parent is absolute we assume 

239 # that the child is a child of the parent unless it uses ".." 

240 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

241 

242 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

243 self.assertIsNone(child.relative_to(parent)) 

244 

245 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

246 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

247 

248 # Test with different netloc 

249 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

250 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True) 

251 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

252 

253 # This is an absolute path so will *always* return a file URI and 

254 # ignore the root parameter. 

255 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True) 

256 self.assertEqual(parent.geturl(), "file:///a/b/c/") 

257 

258 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True) 

259 child = ResourcePath("d/e.txt", root=parent) 

260 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

261 

262 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/"))) 

263 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

264 

265 # Absolute schemeless child with relative parent will always fail. 

266 child = ResourcePath("d/e.txt", root="/a/b/c") 

267 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

268 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

269 

270 def test_parents(self) -> None: 

271 """Test of splitting and parent walking.""" 

272 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

273 child_file = parent.join("subdir/file.txt") 

274 self.assertFalse(child_file.isdir()) 

275 child_subdir, file = child_file.split() 

276 self.assertEqual(file, "file.txt") 

277 self.assertTrue(child_subdir.isdir()) 

278 self.assertEqual(child_file.dirname(), child_subdir) 

279 self.assertEqual(child_file.basename(), file) 

280 self.assertEqual(child_file.parent(), child_subdir) 

281 derived_parent = child_subdir.parent() 

282 self.assertEqual(derived_parent, parent) 

283 self.assertTrue(derived_parent.isdir()) 

284 self.assertEqual(child_file.parent().parent(), parent) 

285 

286 def test_escapes(self) -> None: 

287 """Special characters in file paths""" 

288 src = self.root_uri.join("bbb/???/test.txt") 

289 self.assertNotIn("???", src.path) 

290 self.assertIn("???", src.unquoted_path) 

291 

292 file = src.updatedFile("tests??.txt") 

293 self.assertNotIn("??.txt", file.path) 

294 

295 src = src.updatedFile("tests??.txt") 

296 self.assertIn("??.txt", src.unquoted_path) 

297 

298 # File URI and schemeless URI 

299 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

300 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

301 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

302 

303 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

304 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

305 

306 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

307 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

308 

309 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

310 

311 # dir.join() morphs into a file scheme 

312 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

313 new = dir.join("test_j.txt") 

314 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

315 

316 new2name = "###/test??.txt" 

317 new2 = dir.join(new2name) 

318 self.assertIn("???", new2.unquoted_path) 

319 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

320 

321 fdir = dir.abspath() 

322 self.assertNotIn("???", fdir.path) 

323 self.assertIn("???", fdir.unquoted_path) 

324 self.assertEqual(fdir.scheme, self.scheme) 

325 

326 fnew2 = fdir.join(new2name) 

327 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

328 self.assertNotIn("###", fnew2.path) 

329 

330 # Test that children relative to schemeless and file schemes 

331 # still return the same unquoted name 

332 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

333 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

334 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

335 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

336 

337 # Check for double quoting 

338 plus_path = "/a/b/c+d/" 

339 with self.assertLogs(level="WARNING"): 

340 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

341 self.assertEqual(uri.ospath, plus_path) 

342 

343 # Check that # is not escaped for schemeless URIs 

344 hash_path = "/a/b#/c&d#xyz" 

345 hpos = hash_path.rfind("#") 

346 uri = ResourcePath(hash_path) 

347 self.assertEqual(uri.ospath, hash_path[:hpos]) 

348 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

349 

350 def test_hash(self) -> None: 

351 """Test that we can store URIs in sets and as keys.""" 

352 uri1 = self.root_uri 

353 uri2 = uri1.join("test/") 

354 s = {uri1, uri2} 

355 self.assertIn(uri1, s) 

356 

357 d = {uri1: "1", uri2: "2"} 

358 self.assertEqual(d[uri2], "2") 

359 

360 def test_root_uri(self) -> None: 

361 """Test ResourcePath.root_uri().""" 

362 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

363 self.assertEqual(uri.root_uri().geturl(), self.root) 

364 

365 def test_join(self) -> None: 

366 """Test .join method.""" 

367 root_str = self.root 

368 root = self.root_uri 

369 

370 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

371 add_dir = root.join("b/c/d/") 

372 self.assertTrue(add_dir.isdir()) 

373 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

374 

375 up_relative = root.join("../b/c.txt") 

376 self.assertFalse(up_relative.isdir()) 

377 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

378 

379 quote_example = "hsc/payload/b&c.t@x#t" 

380 needs_quote = root.join(quote_example) 

381 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

382 

383 other = ResourcePath(f"{self.root}test.txt") 

384 self.assertEqual(root.join(other), other) 

385 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

386 

387 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

388 ResourcePath("test.qgraph", forceAbsolute=False) 

389 ) 

390 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

391 

392 qgraph = ResourcePath("test.qgraph") # Absolute URI 

393 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph) 

394 self.assertEqual(joined, qgraph) 

395 

396 def test_quoting(self) -> None: 

397 """Check that quoting works.""" 

398 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

399 subpath = "rootdir/dir1+/file?.txt" 

400 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

401 

402 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

403 self.assertEqual(child.basename(), "file?.txt") 

404 self.assertEqual(child.relativeToPathRoot, subpath) 

405 self.assertIn("%", child.path) 

406 self.assertEqual(child.unquoted_path, "/" + subpath) 

407 

408 def test_ordering(self) -> None: 

409 """Check that greater/less comparison operators work.""" 

410 a = self._make_uri("a.txt") 

411 b = self._make_uri("b/") 

412 self.assertTrue(a < b) 

413 self.assertFalse(a < a) 

414 self.assertTrue(a <= b) 

415 self.assertTrue(a <= a) 

416 self.assertTrue(b > a) 

417 self.assertFalse(b > b) 

418 self.assertTrue(b >= a) 

419 self.assertTrue(b >= b) 

420 

421 

422class GenericReadWriteTestCase(_GenericTestCase): 

423 """Test schemes that can read and write using concrete resources.""" 

424 

425 transfer_modes = ("copy", "move") 

426 testdir: Optional[str] = None 

427 

428 def setUp(self) -> None: 

429 if self.scheme is None: 

430 raise unittest.SkipTest("No scheme defined") 

431 self.root = self._make_uri("") 

432 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

433 

434 if self.scheme == "file": 

435 # Use a local tempdir because on macOS the temp dirs use symlinks 

436 # so relsymlink gets quite confused. 

437 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

438 else: 

439 # Create random tmp directory relative to the test root. 

440 self.tmpdir = self.root_uri.join( 

441 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)), 

442 forceDirectory=True, 

443 ) 

444 self.tmpdir.mkdir() 

445 

446 def tearDown(self) -> None: 

447 if self.tmpdir: 

448 if self.tmpdir.isLocal: 

449 removeTestTempDir(self.tmpdir.ospath) 

450 

451 def test_file(self) -> None: 

452 uri = self.tmpdir.join("test.txt") 

453 self.assertFalse(uri.exists(), f"{uri} should not exist") 

454 self.assertTrue(uri.path.endswith("test.txt")) 

455 

456 content = "abcdefghijklmnopqrstuv\n" 

457 uri.write(content.encode()) 

458 self.assertTrue(uri.exists(), f"{uri} should now exist") 

459 self.assertEqual(uri.read().decode(), content) 

460 self.assertEqual(uri.size(), len(content.encode())) 

461 

462 with self.assertRaises(FileExistsError): 

463 uri.write(b"", overwrite=False) 

464 

465 # Not all backends can tell if a remove fails so we can not 

466 # test that a remove of a non-existent entry is guaranteed to raise. 

467 uri.remove() 

468 self.assertFalse(uri.exists()) 

469 

470 # Ideally the test would remove the file again and raise a 

471 # FileNotFoundError. This is not reliable for remote resources 

472 # and doing an explicit check before trying to remove the resource 

473 # just to raise an exception is deemed an unacceptable overhead. 

474 

475 with self.assertRaises(FileNotFoundError): 

476 uri.read() 

477 

478 with self.assertRaises(FileNotFoundError): 

479 self.tmpdir.join("file/not/there.txt").size() 

480 

481 # Check that creating a URI from a URI returns the same thing 

482 uri2 = ResourcePath(uri) 

483 self.assertEqual(uri, uri2) 

484 self.assertEqual(id(uri), id(uri2)) 

485 

486 def test_mkdir(self) -> None: 

487 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

488 newdir.mkdir() 

489 self.assertTrue(newdir.exists()) 

490 self.assertEqual(newdir.size(), 0) 

491 

492 newfile = newdir.join("temp.txt") 

493 newfile.write("Data".encode()) 

494 self.assertTrue(newfile.exists()) 

495 

496 file = self.tmpdir.join("file.txt") 

497 # Some schemes will realize that the URI is not a file and so 

498 # will raise NotADirectoryError. The file scheme is more permissive 

499 # and lets you write anything but will raise NotADirectoryError 

500 # if a non-directory is already there. We therefore write something 

501 # to the file to ensure that we trigger a portable exception. 

502 file.write(b"") 

503 with self.assertRaises(NotADirectoryError): 

504 file.mkdir() 

505 

506 # The root should exist. 

507 self.root_uri.mkdir() 

508 self.assertTrue(self.root_uri.exists()) 

509 

510 def test_transfer(self) -> None: 

511 src = self.tmpdir.join("test.txt") 

512 content = "Content is some content\nwith something to say\n\n" 

513 src.write(content.encode()) 

514 

515 can_move = "move" in self.transfer_modes 

516 for mode in self.transfer_modes: 

517 if mode == "move": 

518 continue 

519 

520 dest = self.tmpdir.join(f"dest_{mode}.txt") 

521 # Ensure that we get some debugging output. 

522 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

523 dest.transfer_from(src, transfer=mode) 

524 self.assertIn("Transferring ", "\n".join(cm.output)) 

525 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

526 

527 new_content = dest.read().decode() 

528 self.assertEqual(new_content, content) 

529 

530 if mode in ("symlink", "relsymlink"): 

531 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

532 

533 # If the source and destination are hardlinks of each other 

534 # the transfer should work even if overwrite=False. 

535 if mode in ("link", "hardlink"): 

536 dest.transfer_from(src, transfer=mode) 

537 else: 

538 with self.assertRaises( 

539 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

540 ): 

541 dest.transfer_from(src, transfer=mode) 

542 

543 # Transfer again and overwrite. 

544 dest.transfer_from(src, transfer=mode, overwrite=True) 

545 

546 dest.remove() 

547 

548 b = src.read() 

549 self.assertEqual(b.decode(), new_content) 

550 

551 nbytes = 10 

552 subset = src.read(size=nbytes) 

553 self.assertEqual(len(subset), nbytes) 

554 self.assertEqual(subset.decode(), content[:nbytes]) 

555 

556 # Transferring to self should be okay. 

557 src.transfer_from(src, "auto") 

558 

559 with self.assertRaises(ValueError): 

560 src.transfer_from(src, transfer="unknown") 

561 

562 # A move transfer is special. 

563 if can_move: 

564 dest.transfer_from(src, transfer="move") 

565 self.assertFalse(src.exists()) 

566 self.assertTrue(dest.exists()) 

567 else: 

568 src.remove() 

569 

570 dest.remove() 

571 with self.assertRaises(FileNotFoundError): 

572 dest.transfer_from(src, "auto") 

573 

574 def test_local_transfer(self) -> None: 

575 """Test we can transfer to and from local file.""" 

576 remote_src = self.tmpdir.join("src.json") 

577 remote_src.write(b"42") 

578 remote_dest = self.tmpdir.join("dest.json") 

579 

580 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

581 self.assertTrue(tmp.isLocal) 

582 tmp.transfer_from(remote_src, transfer="auto") 

583 self.assertEqual(tmp.read(), remote_src.read()) 

584 

585 remote_dest.transfer_from(tmp, transfer="auto") 

586 self.assertEqual(remote_dest.read(), tmp.read()) 

587 

588 # Temporary (possibly remote) resource. 

589 # Transfers between temporary resources. 

590 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

591 # Temporary local resource. 

592 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

593 remote_tmp.write(b"42") 

594 if not remote_tmp.isLocal: 

595 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

596 with self.assertRaises(RuntimeError): 

597 # Trying to symlink a remote resource is not going 

598 # to work. A hardlink could work but would rely 

599 # on the local temp space being on the same 

600 # filesystem as the target. 

601 local_tmp.transfer_from(remote_tmp, transfer) 

602 local_tmp.transfer_from(remote_tmp, "move") 

603 self.assertFalse(remote_tmp.exists()) 

604 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

605 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

606 

607 # Transfer of missing remote. 

608 remote_tmp.remove() 

609 with self.assertRaises(FileNotFoundError): 

610 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

611 

612 def test_local(self) -> None: 

613 """Check that remote resources can be made local.""" 

614 src = self.tmpdir.join("test.txt") 

615 original_content = "Content is some content\nwith something to say\n\n" 

616 src.write(original_content.encode()) 

617 

618 # Run this twice to ensure use of cache in code coverage 

619 # if applicable. 

620 for _ in (1, 2): 

621 with src.as_local() as local_uri: 

622 self.assertTrue(local_uri.isLocal) 

623 content = local_uri.read().decode() 

624 self.assertEqual(content, original_content) 

625 

626 if src.isLocal: 

627 self.assertEqual(src, local_uri) 

628 

629 with self.assertRaises(IsADirectoryError): 

630 with self.root_uri.as_local() as local_uri: 

631 pass 

632 

633 def test_walk(self) -> None: 

634 """Walk a directory hierarchy.""" 

635 root = self.tmpdir.join("walk/") 

636 

637 # Look for a file that is not there 

638 file = root.join("config/basic/butler.yaml") 

639 found_list = list(ResourcePath.findFileResources([file])) 

640 self.assertEqual(found_list[0], file) 

641 

642 # First create the files (content is irrelevant). 

643 expected_files = { 

644 "dir1/a.yaml", 

645 "dir1/b.yaml", 

646 "dir1/c.json", 

647 "dir2/d.json", 

648 "dir2/e.yaml", 

649 } 

650 expected_uris = {root.join(f) for f in expected_files} 

651 for uri in expected_uris: 

652 uri.write(b"") 

653 self.assertTrue(uri.exists()) 

654 

655 # Look for the files. 

656 found = set(ResourcePath.findFileResources([root])) 

657 self.assertEqual(found, expected_uris) 

658 

659 # Now solely the YAML files. 

660 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

661 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

662 self.assertEqual(found, expected_yaml) 

663 

664 # Now two explicit directories and a file 

665 expected = set(u for u in expected_yaml) 

666 expected.add(file) 

667 

668 found = set( 

669 ResourcePath.findFileResources( 

670 [file, root.join("dir1/"), root.join("dir2/")], 

671 file_filter=r".*\.yaml$", 

672 ) 

673 ) 

674 self.assertEqual(found, expected) 

675 

676 # Group by directory -- find everything and compare it with what 

677 # we expected to be there in total. 

678 found_yaml = set() 

679 counter = 0 

680 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True): 

681 assert not isinstance(uris, ResourcePath) # for mypy. 

682 found_uris = set(uris) 

683 if found_uris: 

684 counter += 1 

685 

686 found_yaml.update(found_uris) 

687 

688 expected_yaml_2 = expected_yaml 

689 expected_yaml_2.add(file) 

690 self.assertEqual(found_yaml, expected_yaml) 

691 self.assertEqual(counter, 3) 

692 

693 # Grouping but check that single files are returned in a single group 

694 # at the end 

695 file2 = root.join("config/templates/templates-bad.yaml") 

696 found_grouped = [ 

697 [uri for uri in group] 

698 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True) 

699 if not isinstance(group, ResourcePath) # For mypy. 

700 ] 

701 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}") 

702 self.assertEqual(list(found_grouped[1]), [file, file2]) 

703 

704 with self.assertRaises(ValueError): 

705 # The list forces the generator to run. 

706 list(file.walk()) 

707 

708 # A directory that does not exist returns nothing. 

709 self.assertEqual(list(root.join("dir3/").walk()), []) 

710 

711 def test_large_walk(self) -> None: 

712 # In some systems pagination is used so ensure that we can handle 

713 # large numbers of files. For example S3 limits us to 1000 responses 

714 # per listing call. 

715 created = set() 

716 counter = 1 

717 n_dir1 = 1100 

718 root = self.tmpdir.join("large_walk", forceDirectory=True) 

719 while counter <= n_dir1: 

720 new = ResourcePath(root.join(f"file{counter:04d}.txt")) 

721 new.write(f"{counter}".encode())