Coverage for python/lsst/resources/tests.py: 11%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

479 statements  

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import unittest 

19import urllib.parse 

20import uuid 

21from typing import Any, Callable, Iterable, Optional, Union 

22 

23from lsst.resources import ResourcePath 

24from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

25 

26TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

27 

28 

29def _check_open( 

30 test_case: Union[_GenericTestCase, unittest.TestCase], 

31 uri: ResourcePath, 

32 *, 

33 mode_suffixes: Iterable[str] = ("", "t", "b"), 

34 **kwargs: Any, 

35) -> None: 

36 """Test an implementation of ButlerURI.open. 

37 

38 Parameters 

39 ---------- 

40 test_case : `unittest.TestCase` 

41 Test case to use for assertions. 

42 uri : `ResourcePath` 

43 URI to use for tests. Must point to a writeable location that is not 

44 yet occupied by a file. On return, the location may point to a file 

45 only if the test fails. 

46 mode_suffixes : `Iterable` of `str` 

47 Suffixes to pass as part of the ``mode`` argument to 

48 `ResourcePath.open`, indicating whether to open as binary or as text; 

49 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

50 **kwargs 

51 Additional keyword arguments to forward to all calls to `open`. 

52 """ 

53 text_content = "wxyz🙂" 

54 bytes_content = uuid.uuid4().bytes 

55 content_by_mode_suffix = { 

56 "": text_content, 

57 "t": text_content, 

58 "b": bytes_content, 

59 } 

60 empty_content_by_mode_suffix = { 

61 "": "", 

62 "t": "", 

63 "b": b"", 

64 } 

65 # To appease mypy 

66 double_content_by_mode_suffix = { 

67 "": text_content + text_content, 

68 "t": text_content + text_content, 

69 "b": bytes_content + bytes_content, 

70 } 

71 for mode_suffix in mode_suffixes: 

72 content = content_by_mode_suffix[mode_suffix] 

73 double_content = double_content_by_mode_suffix[mode_suffix] 

74 # Create file with mode='x', which prohibits overwriting. 

75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

76 write_buffer.write(content) 

77 test_case.assertTrue(uri.exists()) 

78 # Check that opening with 'x' now raises, and does not modify content. 

79 with test_case.assertRaises(FileExistsError): 

80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

81 write_buffer.write("bad") 

82 # Read the file we created and check the contents. 

83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

84 test_case.assertEqual(read_buffer.read(), content) 

85 # Write two copies of the content, overwriting the single copy there. 

86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

87 write_buffer.write(double_content) 

88 # Read again, this time use mode='r+', which reads what is there and 

89 # then lets us write more; we'll use that to reset the file to one 

90 # copy of the content. 

91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

92 test_case.assertEqual(rw_buffer.read(), double_content) 

93 rw_buffer.seek(0) 

94 rw_buffer.truncate() 

95 rw_buffer.write(content) 

96 rw_buffer.seek(0) 

97 test_case.assertEqual(rw_buffer.read(), content) 

98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

99 test_case.assertEqual(read_buffer.read(), content) 

100 # Append some more content to the file; should now have two copies. 

101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

102 append_buffer.write(content) 

103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

104 test_case.assertEqual(read_buffer.read(), double_content) 

105 # Final mode to check is w+, which does read/write but truncates first. 

106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

108 rw_buffer.write(content) 

109 rw_buffer.seek(0) 

110 test_case.assertEqual(rw_buffer.read(), content) 

111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

112 test_case.assertEqual(read_buffer.read(), content) 

113 # Remove file to make room for the next loop of tests with this URI. 

114 uri.remove() 

115 

116 

117class _GenericTestCase: 

118 """Generic base class for test mixin.""" 

119 

120 scheme: Optional[str] = None 

121 netloc: Optional[str] = None 

122 path1 = "test_dir" 

123 path2 = "file.txt" 

124 

125 # Because we use a mixin for tests mypy needs to understand that 

126 # the unittest.TestCase methods exist. 

127 # We do not inherit from unittest.TestCase because that results 

128 # in the tests defined here being run as well as the tests in the 

129 # test file itself. We can make those tests skip but it gives an 

130 # uniformative view of how many tests are running. 

131 assertEqual: Callable 

132 assertNotEqual: Callable 

133 assertIsNone: Callable 

134 assertIn: Callable 

135 assertNotIn: Callable 

136 assertFalse: Callable 

137 assertTrue: Callable 

138 assertRaises: Callable 

139 assertLogs: Callable 

140 

141 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str: 

142 if self.scheme is not None: 

143 if netloc is None: 

144 netloc = self.netloc 

145 if path.startswith("/"): 

146 path = path[1:] 

147 return f"{self.scheme}://{netloc}/{path}" 

148 else: 

149 return path 

150 

151 

152class GenericTestCase(_GenericTestCase): 

153 """Test cases for generic manipulation of a `ResourcePath`""" 

154 

155 def setUp(self) -> None: 

156 if self.scheme is None: 

157 raise unittest.SkipTest("No scheme defined") 

158 self.root = self._make_uri("") 

159 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

160 

161 def test_creation(self) -> None: 

162 self.assertEqual(self.root_uri.scheme, self.scheme) 

163 self.assertEqual(self.root_uri.netloc, self.netloc) 

164 self.assertFalse(self.root_uri.query) 

165 self.assertFalse(self.root_uri.params) 

166 

167 with self.assertRaises(ValueError): 

168 ResourcePath({}) # type: ignore 

169 

170 with self.assertRaises(RuntimeError): 

171 ResourcePath(self.root_uri, isTemporary=True) 

172 

173 file = self.root_uri.join("file.txt") 

174 with self.assertRaises(RuntimeError): 

175 ResourcePath(file, forceDirectory=True) 

176 

177 with self.assertRaises(NotImplementedError): 

178 ResourcePath("unknown://netloc") 

179 

180 replaced = file.replace(fragment="frag") 

181 self.assertEqual(replaced.fragment, "frag") 

182 

183 with self.assertRaises(ValueError): 

184 file.replace(scheme="new") 

185 

186 self.assertNotEqual(replaced, str(replaced)) 

187 self.assertNotEqual(str(replaced), replaced) 

188 

189 def test_extension(self) -> None: 

190 uri = ResourcePath(self._make_uri("dir/test.txt")) 

191 self.assertEqual(uri.updatedExtension(None), uri) 

192 self.assertEqual(uri.updatedExtension(".txt"), uri) 

193 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

194 

195 fits = uri.updatedExtension(".fits.gz") 

196 self.assertEqual(fits.basename(), "test.fits.gz") 

197 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

198 

199 extensionless = self.root_uri.join("no_ext") 

200 self.assertEqual(extensionless.getExtension(), "") 

201 extension = extensionless.updatedExtension(".fits") 

202 self.assertEqual(extension.getExtension(), ".fits") 

203 

204 def test_relative(self) -> None: 

205 """Check that we can get subpaths back from two URIs""" 

206 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

207 self.assertTrue(parent.isdir()) 

208 child = parent.join("dir1/file.txt") 

209 

210 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

211 

212 not_child = ResourcePath("/a/b/dir1/file.txt") 

213 self.assertIsNone(not_child.relative_to(parent)) 

214 self.assertFalse(not_child.isdir()) 

215 

216 not_directory = parent.join("dir1/file2.txt") 

217 self.assertIsNone(child.relative_to(not_directory)) 

218 

219 # Relative URIs 

220 parent = ResourcePath("a/b/", forceAbsolute=False) 

221 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

222 self.assertFalse(child.scheme) 

223 self.assertEqual(child.relative_to(parent), "c/d.txt") 

224 

225 # forceAbsolute=True should work even on an existing ResourcePath 

226 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

227 

228 # Absolute URI and schemeless URI 

229 parent = self.root_uri.join("/a/b/c/") 

230 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

231 

232 # If the child is relative and the parent is absolute we assume 

233 # that the child is a child of the parent unless it uses ".." 

234 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

235 

236 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

237 self.assertIsNone(child.relative_to(parent)) 

238 

239 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

240 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

241 

242 # Test with different netloc 

243 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

244 parent = ResourcePath(self._make_uri("a", netloc="other")) 

245 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

246 

247 # Schemeless absolute child. 

248 # Schemeless absolute URI is constructed using root= parameter. 

249 # For now the root parameter can not be anything other than a file. 

250 if self.scheme == "file": 

251 parent = ResourcePath("/a/b/c", root=self.root_uri) 

252 child = ResourcePath("d/e.txt", root=parent) 

253 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

254 

255 parent = ResourcePath("c/", root="/a/b/") 

256 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

257 

258 # Absolute schemeless child with relative parent will always fail. 

259 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

260 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

261 

262 def test_parents(self) -> None: 

263 """Test of splitting and parent walking.""" 

264 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

265 child_file = parent.join("subdir/file.txt") 

266 self.assertFalse(child_file.isdir()) 

267 child_subdir, file = child_file.split() 

268 self.assertEqual(file, "file.txt") 

269 self.assertTrue(child_subdir.isdir()) 

270 self.assertEqual(child_file.dirname(), child_subdir) 

271 self.assertEqual(child_file.basename(), file) 

272 self.assertEqual(child_file.parent(), child_subdir) 

273 derived_parent = child_subdir.parent() 

274 self.assertEqual(derived_parent, parent) 

275 self.assertTrue(derived_parent.isdir()) 

276 self.assertEqual(child_file.parent().parent(), parent) 

277 

278 def test_escapes(self) -> None: 

279 """Special characters in file paths""" 

280 src = self.root_uri.join("bbb/???/test.txt") 

281 self.assertNotIn("???", src.path) 

282 self.assertIn("???", src.unquoted_path) 

283 

284 file = src.updatedFile("tests??.txt") 

285 self.assertNotIn("??.txt", file.path) 

286 

287 src = src.updatedFile("tests??.txt") 

288 self.assertIn("??.txt", src.unquoted_path) 

289 

290 # File URI and schemeless URI 

291 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

292 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

293 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

294 

295 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

296 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

297 

298 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

299 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

300 

301 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

302 

303 # dir.join() morphs into a file scheme 

304 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

305 new = dir.join("test_j.txt") 

306 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

307 

308 new2name = "###/test??.txt" 

309 new2 = dir.join(new2name) 

310 self.assertIn("???", new2.unquoted_path) 

311 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

312 

313 fdir = dir.abspath() 

314 self.assertNotIn("???", fdir.path) 

315 self.assertIn("???", fdir.unquoted_path) 

316 self.assertEqual(fdir.scheme, self.scheme) 

317 

318 fnew2 = fdir.join(new2name) 

319 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

320 self.assertNotIn("###", fnew2.path) 

321 

322 # Test that children relative to schemeless and file schemes 

323 # still return the same unquoted name 

324 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

325 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

326 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

327 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

328 

329 # Check for double quoting 

330 plus_path = "/a/b/c+d/" 

331 with self.assertLogs(level="WARNING"): 

332 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

333 self.assertEqual(uri.ospath, plus_path) 

334 

335 # Check that # is not escaped for schemeless URIs 

336 hash_path = "/a/b#/c&d#xyz" 

337 hpos = hash_path.rfind("#") 

338 uri = ResourcePath(hash_path) 

339 self.assertEqual(uri.ospath, hash_path[:hpos]) 

340 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

341 

342 def test_hash(self) -> None: 

343 """Test that we can store URIs in sets and as keys.""" 

344 uri1 = self.root_uri 

345 uri2 = uri1.join("test/") 

346 s = {uri1, uri2} 

347 self.assertIn(uri1, s) 

348 

349 d = {uri1: "1", uri2: "2"} 

350 self.assertEqual(d[uri2], "2") 

351 

352 def test_root_uri(self) -> None: 

353 """Test ResourcePath.root_uri().""" 

354 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

355 self.assertEqual(uri.root_uri().geturl(), self.root) 

356 

357 def test_join(self) -> None: 

358 """Test .join method.""" 

359 root_str = self.root 

360 root = self.root_uri 

361 

362 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

363 add_dir = root.join("b/c/d/") 

364 self.assertTrue(add_dir.isdir()) 

365 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

366 

367 up_relative = root.join("../b/c.txt") 

368 self.assertFalse(up_relative.isdir()) 

369 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

370 

371 quote_example = "hsc/payload/b&c.t@x#t" 

372 needs_quote = root.join(quote_example) 

373 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

374 

375 other = ResourcePath(f"{self.root}test.txt") 

376 self.assertEqual(root.join(other), other) 

377 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

378 

379 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

380 ResourcePath("test.qgraph", forceAbsolute=False) 

381 ) 

382 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

383 

384 with self.assertRaises(ValueError): 

385 ResourcePath(f"{self.root}hsc/payload/").join(ResourcePath("test.qgraph")) 

386 

387 def test_quoting(self) -> None: 

388 """Check that quoting works.""" 

389 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

390 subpath = "rootdir/dir1+/file?.txt" 

391 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

392 

393 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

394 self.assertEqual(child.basename(), "file?.txt") 

395 self.assertEqual(child.relativeToPathRoot, subpath) 

396 self.assertIn("%", child.path) 

397 self.assertEqual(child.unquoted_path, "/" + subpath) 

398 

399 

400class GenericReadWriteTestCase(_GenericTestCase): 

401 """Test schemes that can read and write using concrete resources.""" 

402 

403 transfer_modes = ("copy", "move") 

404 testdir: Optional[str] = None 

405 

406 def setUp(self) -> None: 

407 if self.scheme is None: 

408 raise unittest.SkipTest("No scheme defined") 

409 self.root = self._make_uri("") 

410 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

411 

412 if self.scheme == "file": 

413 # Use a local tempdir because on macOS the temp dirs use symlinks 

414 # so relsymlink gets quite confused. 

415 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

416 else: 

417 # Create tmp directory relative to the test root. 

418 self.tmpdir = self.root_uri.join("TESTING/") 

419 self.tmpdir.mkdir() 

420 

421 def tearDown(self) -> None: 

422 if self.tmpdir: 

423 if self.tmpdir.isLocal: 

424 removeTestTempDir(self.tmpdir.ospath) 

425 

426 def test_file(self) -> None: 

427 uri = self.tmpdir.join("test.txt") 

428 self.assertFalse(uri.exists(), f"{uri} should not exist") 

429 self.assertTrue(uri.path.endswith("test.txt")) 

430 

431 content = "abcdefghijklmnopqrstuv\n" 

432 uri.write(content.encode()) 

433 self.assertTrue(uri.exists(), f"{uri} should now exist") 

434 self.assertEqual(uri.read().decode(), content) 

435 self.assertEqual(uri.size(), len(content.encode())) 

436 

437 with self.assertRaises(FileExistsError): 

438 uri.write(b"", overwrite=False) 

439 

440 # Not all backends can tell if a remove fails so we can not 

441 # test that a remove of a non-existent entry is guaranteed to raise. 

442 uri.remove() 

443 self.assertFalse(uri.exists()) 

444 

445 # Ideally the test would remove the file again and raise a 

446 # FileNotFoundError. This is not reliable for remote resources 

447 # and doing an explicit check before trying to remove the resource 

448 # just to raise an exception is deemed an unacceptable overhead. 

449 

450 with self.assertRaises(FileNotFoundError): 

451 uri.read() 

452 

453 with self.assertRaises(FileNotFoundError): 

454 self.tmpdir.join("file/not/there.txt").size() 

455 

456 # Check that creating a URI from a URI returns the same thing 

457 uri2 = ResourcePath(uri) 

458 self.assertEqual(uri, uri2) 

459 self.assertEqual(id(uri), id(uri2)) 

460 

461 def test_mkdir(self) -> None: 

462 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

463 newdir.mkdir() 

464 self.assertTrue(newdir.exists()) 

465 self.assertEqual(newdir.size(), 0) 

466 

467 newfile = newdir.join("temp.txt") 

468 newfile.write("Data".encode()) 

469 self.assertTrue(newfile.exists()) 

470 

471 file = self.tmpdir.join("file.txt") 

472 # Some schemes will realize that the URI is not a file and so 

473 # will raise NotADirectoryError. The file scheme is more permissive 

474 # and lets you write anything but will raise NotADirectoryError 

475 # if a non-directory is already there. We therefore write something 

476 # to the file to ensure that we trigger a portable exception. 

477 file.write(b"") 

478 with self.assertRaises(NotADirectoryError): 

479 file.mkdir() 

480 

481 # The root should exist. 

482 self.root_uri.mkdir() 

483 self.assertTrue(self.root_uri.exists()) 

484 

485 def test_transfer(self) -> None: 

486 src = self.tmpdir.join("test.txt") 

487 content = "Content is some content\nwith something to say\n\n" 

488 src.write(content.encode()) 

489 

490 can_move = "move" in self.transfer_modes 

491 for mode in self.transfer_modes: 

492 if mode == "move": 

493 continue 

494 

495 dest = self.tmpdir.join(f"dest_{mode}.txt") 

496 # Ensure that we get some debugging output. 

497 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

498 dest.transfer_from(src, transfer=mode) 

499 self.assertIn("Transferring ", "\n".join(cm.output)) 

500 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

501 

502 new_content = dest.read().decode() 

503 self.assertEqual(new_content, content) 

504 

505 if mode in ("symlink", "relsymlink"): 

506 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

507 

508 # If the source and destination are hardlinks of each other 

509 # the transfer should work even if overwrite=False. 

510 if mode in ("link", "hardlink"): 

511 dest.transfer_from(src, transfer=mode) 

512 else: 

513 with self.assertRaises( 

514 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

515 ): 

516 dest.transfer_from(src, transfer=mode) 

517 

518 # Transfer again and overwrite. 

519 dest.transfer_from(src, transfer=mode, overwrite=True) 

520 

521 dest.remove() 

522 

523 b = src.read() 

524 self.assertEqual(b.decode(), new_content) 

525 

526 nbytes = 10 

527 subset = src.read(size=nbytes) 

528 self.assertEqual(len(subset), nbytes) 

529 self.assertEqual(subset.decode(), content[:nbytes]) 

530 

531 # Transferring to self should be okay. 

532 src.transfer_from(src, "auto") 

533 

534 with self.assertRaises(ValueError): 

535 src.transfer_from(src, transfer="unknown") 

536 

537 # A move transfer is special. 

538 if can_move: 

539 dest.transfer_from(src, transfer="move") 

540 self.assertFalse(src.exists()) 

541 self.assertTrue(dest.exists()) 

542 else: 

543 src.remove() 

544 

545 dest.remove() 

546 with self.assertRaises(FileNotFoundError): 

547 dest.transfer_from(src, "auto") 

548 

549 def test_local_transfer(self) -> None: 

550 """Test we can transfer to and from local file.""" 

551 remote_src = self.tmpdir.join("src.json") 

552 remote_src.write(b"42") 

553 remote_dest = self.tmpdir.join("dest.json") 

554 

555 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

556 self.assertTrue(tmp.isLocal) 

557 tmp.transfer_from(remote_src, transfer="auto") 

558 self.assertEqual(tmp.read(), remote_src.read()) 

559 

560 remote_dest.transfer_from(tmp, transfer="auto") 

561 self.assertEqual(remote_dest.read(), tmp.read()) 

562 

563 # Temporary (possibly remote) resource. 

564 # Transfers between temporary resources. 

565 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

566 # Temporary local resource. 

567 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

568 remote_tmp.write(b"42") 

569 if not remote_tmp.isLocal: 

570 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

571 with self.assertRaises(RuntimeError): 

572 # Trying to symlink a remote resource is not going 

573 # to work. A hardlink could work but would rely 

574 # on the local temp space being on the same 

575 # filesystem as the target. 

576 local_tmp.transfer_from(remote_tmp, transfer) 

577 local_tmp.transfer_from(remote_tmp, "move") 

578 self.assertFalse(remote_tmp.exists()) 

579 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

580 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

581 

582 # Transfer of missing remote. 

583 remote_tmp.remove() 

584 with self.assertRaises(FileNotFoundError): 

585 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

586 

587 def test_local(self) -> None: 

588 """Check that remote resources can be made local.""" 

589 src = self.tmpdir.join("test.txt") 

590 original_content = "Content is some content\nwith something to say\n\n" 

591 src.write(original_content.encode()) 

592 

593 # Run this twice to ensure use of cache in code coverage 

594 # if applicable. 

595 for _ in (1, 2): 

596 with src.as_local() as local_uri: 

597 self.assertTrue(local_uri.isLocal) 

598 content = local_uri.read().decode() 

599 self.assertEqual(content, original_content) 

600 

601 if src.isLocal: 

602 self.assertEqual(src, local_uri) 

603 

604 with self.assertRaises(IsADirectoryError): 

605 with self.root_uri.as_local() as local_uri: 

606 pass 

607 

608 def test_walk(self) -> None: 

609 """Walk a directory hierarchy.""" 

610 root = self.tmpdir.join("walk/") 

611 

612 # Look for a file that is not there 

613 file = root.join("config/basic/butler.yaml") 

614 found_list = list(ResourcePath.findFileResources([file])) 

615 self.assertEqual(found_list[0], file) 

616 

617 # First create the files (content is irrelevant). 

618 expected_files = { 

619 "dir1/a.yaml", 

620 "dir1/b.yaml", 

621 "dir1/c.json", 

622 "dir2/d.json", 

623 "dir2/e.yaml", 

624 } 

625 expected_uris = {root.join(f) for f in expected_files} 

626 for uri in expected_uris: 

627 uri.write(b"") 

628 self.assertTrue(uri.exists()) 

629 

630 # Look for the files. 

631 found = set(ResourcePath.findFileResources([root])) 

632 self.assertEqual(found, expected_uris) 

633 

634 # Now solely the YAML files. 

635 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

636 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

637 self.assertEqual(found, expected_yaml) 

638 

639 # Now two explicit directories and a file 

640 expected = set(u for u in expected_yaml) 

641 expected.add(file) 

642 

643 found = set( 

644 ResourcePath.findFileResources( 

645 [file, root.join("dir1/"), root.join("dir2/")], 

646 file_filter=r".*\.yaml$", 

647 ) 

648 ) 

649 self.assertEqual(found, expected) 

650 

651 # Group by directory -- find everything and compare it with what 

652 # we expected to be there in total. 

653 found_yaml = set() 

654 counter = 0 

655 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True): 

656 assert not isinstance(uris, ResourcePath) # for mypy. 

657 found_uris = set(uris) 

658 if found_uris: 

659 counter += 1 

660 

661 found_yaml.update(found_uris) 

662 

663 expected_yaml_2 = expected_yaml 

664 expected_yaml_2.add(file) 

665 self.assertEqual(found_yaml, expected_yaml) 

666 self.assertEqual(counter, 3) 

667 

668 # Grouping but check that single files are returned in a single group 

669 # at the end 

670 file2 = root.join("config/templates/templates-bad.yaml") 

671 found_grouped = [ 

672 [uri for uri in group] 

673 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True) 

674 if not isinstance(group, ResourcePath) # For mypy. 

675 ] 

676 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}") 

677 self.assertEqual(list(found_grouped[1]), [file, file2]) 

678 

679 with self.assertRaises(ValueError): 

680 # The list forces the generator to run. 

681 list(file.walk()) 

682 

683 # A directory that does not exist returns nothing. 

684 self.assertEqual(list(root.join("dir3/").walk()), []) 

685 

686 def test_large_walk(self) -> None: 

687 # In some systems pagination is used so ensure that we can handle 

688 # large numbers of files. For example S3 limits us to 1000 responses 

689 # per listing call. 

690 created = set() 

691 counter = 1 

692 n_dir1 = 1100 

693 root = self.tmpdir.join("large_walk", forceDirectory=True) 

694 while counter <= n_dir1: 

695 new = ResourcePath(root.join(f"file{counter:04d}.txt")) 

696 new.write(f"{counter}".encode()) 

697 created.add(new) 

698 counter += 1 

699 counter = 1 

700 # Put some in a subdirectory to make sure we are looking in a 

701 # hierarchy. 

702 n_dir2 = 100 

703 subdir = root.join("subdir", forceDirectory=True) 

704 while counter <= n_dir2: 

705 new = ResourcePath(subdir.join(f"file{counter:04d}.txt")) 

706 new.write(f"{counter}".encode()) 

707 created.add(new) 

708 counter += 1 

709 

710 found = set(ResourcePath.findFileResources([root])) 

711 self.assertEqual(len(found), n_dir1 + n_dir2) 

712 self.assertEqual(found, created) 

713 

714 # Again with grouping. 

715 # (mypy gets upset not knowing which of the two options is being 

716 # returned so add useless instance check). 

717 found_list = list( 

718 [uri for uri in group] 

719 for group in ResourcePath.findFileResources([root], grouped=True) 

720 if not isinstance(group, ResourcePath) # For mypy. 

721 ) 

722 self.assertEqual(len(found_list), 2) 

723 self.assertEqual(len(found_list[0]), n_dir1) 

724 self.assertEqual(len(found_list[1]), n_dir2) 

725 

726 def test_temporary(self) -> None: 

727 prefix = self.tmpdir.join("tmp", forceDirectory=True) 

728 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp: 

729 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

730 self.assertTrue(tmp.isabs(), f"uri: {tmp}") 

731 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

732 tmp.write(b"abcd") 

733 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

734 self.assertTrue(tmp.isTemporary) 

735 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

736 

737 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

738 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp: 

739 # Use a specified tmpdir and check it is okay for the file 

740 # to not be created. 

741 self.assertFalse(tmp.getExtension()) 

742 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

743 self.assertEqual(tmp.scheme, self.scheme) 

744 self.assertTrue(tmp.isTemporary) 

745 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists") 

746 

747 # Fake a directory suffix. 

748 with self.assertRaises(NotImplementedError): 

749 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp: 

750 pass 

751 

752 def test_open(self) -> None: 

753 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

754 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp: 

755 _check_open(self, tmp, mode_suffixes=("", "t")) 

756 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16") 

757 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True) 

758 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True) 

759 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp: 

760 _check_open(self, tmp, mode_suffixes=("b",)) 

761 _check_open(self, tmp, mode_suffixes=("b"), prefer_file_temporary=True) 

762 

763 with self.assertRaises(IsADirectoryError): 

764 with self.root_uri.open(): 

765 pass 

766 

767 def test_mexists(self) -> None: 

768 root = self.tmpdir.join("mexists/") 

769 

770 # A file that is not there. 

771 file = root.join("config/basic/butler.yaml") 

772 

773 # Create some files. 

774 expected_files = { 

775 "dir1/a.yaml", 

776 "dir1/b.yaml", 

777 "dir2/e.yaml", 

778 } 

779 expected_uris = {root.join(f) for f in expected_files} 

780 for uri in expected_uris: 

781 uri.write(b"") 

782 self.assertTrue(uri.exists()) 

783 expected_uris.add(file) 

784 

785 multi = ResourcePath.mexists(expected_uris) 

786 

787 for uri, is_there in multi.items(): 

788 if uri == file: 

789 self.assertFalse(is_there) 

790 else: 

791 self.assertTrue(is_there)