Coverage for python/lsst/resources/tests.py: 7%

545 statements  

« prev     ^ index     » next       coverage.py v7.4.3, created at 2024-03-13 09:59 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import random 

19import string 

20import unittest 

21import urllib.parse 

22import uuid 

23from collections.abc import Iterable 

24from typing import TYPE_CHECKING, Any 

25 

26from lsst.resources import ResourcePath 

27from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

28 

29TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

30 

31 

32def _check_open( 

33 test_case: _GenericTestCase | unittest.TestCase, 

34 uri: ResourcePath, 

35 *, 

36 mode_suffixes: Iterable[str] = ("", "t", "b"), 

37 **kwargs: Any, 

38) -> None: 

39 """Test an implementation of ButlerURI.open. 

40 

41 Parameters 

42 ---------- 

43 test_case : `unittest.TestCase` 

44 Test case to use for assertions. 

45 uri : `ResourcePath` 

46 URI to use for tests. Must point to a writeable location that is not 

47 yet occupied by a file. On return, the location may point to a file 

48 only if the test fails. 

49 mode_suffixes : `~collections.abc.Iterable` of `str` 

50 Suffixes to pass as part of the ``mode`` argument to 

51 `ResourcePath.open`, indicating whether to open as binary or as text; 

52 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

53 **kwargs 

54 Additional keyword arguments to forward to all calls to `open`. 

55 """ 

56 text_content = "abcdefghijklmnopqrstuvwxyz🙂" 

57 bytes_content = uuid.uuid4().bytes 

58 content_by_mode_suffix = { 

59 "": text_content, 

60 "t": text_content, 

61 "b": bytes_content, 

62 } 

63 empty_content_by_mode_suffix = { 

64 "": "", 

65 "t": "", 

66 "b": b"", 

67 } 

68 # To appease mypy 

69 double_content_by_mode_suffix = { 

70 "": text_content + text_content, 

71 "t": text_content + text_content, 

72 "b": bytes_content + bytes_content, 

73 } 

74 for mode_suffix in mode_suffixes: 

75 content = content_by_mode_suffix[mode_suffix] 

76 double_content = double_content_by_mode_suffix[mode_suffix] 

77 # Create file with mode='x', which prohibits overwriting. 

78 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

79 write_buffer.write(content) 

80 test_case.assertTrue(uri.exists()) 

81 # Check that opening with 'x' now raises, and does not modify content. 

82 with test_case.assertRaises(FileExistsError): 

83 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

84 write_buffer.write("bad") 

85 # Read the file we created and check the contents. 

86 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

87 test_case.assertEqual(read_buffer.read(), content) 

88 # Check that we can read bytes in a loop and get EOF 

89 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

90 # Seek off the end of the file and should read empty back. 

91 read_buffer.seek(1024) 

92 test_case.assertEqual(read_buffer.tell(), 1024) 

93 content_read = read_buffer.read() # Read as much as we can. 

94 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

95 

96 # First read more than the content. 

97 read_buffer.seek(0) 

98 size = len(content) * 3 

99 chunk_read = read_buffer.read(size) 

100 test_case.assertEqual(chunk_read, content) 

101 

102 # Repeated reads should always return empty string. 

103 chunk_read = read_buffer.read(size) 

104 test_case.assertEqual(len(chunk_read), 0) 

105 chunk_read = read_buffer.read(size) 

106 test_case.assertEqual(len(chunk_read), 0) 

107 

108 # Go back to start of file and read in smaller chunks. 

109 read_buffer.seek(0) 

110 size = len(content) // 3 

111 

112 content_read = empty_content_by_mode_suffix[mode_suffix] 

113 n_reads = 0 

114 while chunk_read := read_buffer.read(size): 

115 content_read += chunk_read 

116 n_reads += 1 

117 if n_reads > 10: # In case EOF never hits because of bug. 

118 raise AssertionError( 

119 f"Failed to stop reading from file after {n_reads} loops. " 

120 f"Read {len(content_read)} bytes/characters. Expected {len(content)}." 

121 ) 

122 test_case.assertEqual(content_read, content) 

123 

124 # Go back to start of file and read the entire thing. 

125 read_buffer.seek(0) 

126 content_read = read_buffer.read() 

127 test_case.assertEqual(content_read, content) 

128 

129 # Seek off the end of the file and should read empty back. 

130 # We run this check twice since in some cases the handle will 

131 # cache knowledge of the file size. 

132 read_buffer.seek(1024) 

133 test_case.assertEqual(read_buffer.tell(), 1024) 

134 content_read = read_buffer.read() 

135 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

136 

137 # Write two copies of the content, overwriting the single copy there. 

138 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

139 write_buffer.write(double_content) 

140 # Read again, this time use mode='r+', which reads what is there and 

141 # then lets us write more; we'll use that to reset the file to one 

142 # copy of the content. 

143 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

144 test_case.assertEqual(rw_buffer.read(), double_content) 

145 rw_buffer.seek(0) 

146 rw_buffer.truncate() 

147 rw_buffer.write(content) 

148 rw_buffer.seek(0) 

149 test_case.assertEqual(rw_buffer.read(), content) 

150 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

151 test_case.assertEqual(read_buffer.read(), content) 

152 # Append some more content to the file; should now have two copies. 

153 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

154 append_buffer.write(content) 

155 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

156 test_case.assertEqual(read_buffer.read(), double_content) 

157 # Final mode to check is w+, which does read/write but truncates first. 

158 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

159 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

160 rw_buffer.write(content) 

161 rw_buffer.seek(0) 

162 test_case.assertEqual(rw_buffer.read(), content) 

163 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

164 test_case.assertEqual(read_buffer.read(), content) 

165 # Remove file to make room for the next loop of tests with this URI. 

166 uri.remove() 

167 

168 

169if TYPE_CHECKING: 

170 

171 class TestCaseMixin(unittest.TestCase): 

172 """Base class for mixin test classes that use TestCase methods.""" 

173 

174 pass 

175 

176else: 

177 

178 class TestCaseMixin: 

179 """Do-nothing definition of mixin base class for regular execution.""" 

180 

181 pass 

182 

183 

184class _GenericTestCase(TestCaseMixin): 

185 """Generic base class for test mixin.""" 

186 

187 scheme: str | None = None 

188 netloc: str | None = None 

189 base_path: str | None = None 

190 path1 = "test_dir" 

191 path2 = "file.txt" 

192 

193 def _make_uri(self, path: str, netloc: str | None = None) -> str: 

194 if self.scheme is not None: 

195 if netloc is None: 

196 netloc = self.netloc 

197 if path.startswith("/"): 

198 path = path[1:] 

199 if self.base_path is not None: 

200 path = f"{self.base_path}/{path}".lstrip("/") 

201 

202 return f"{self.scheme}://{netloc}/{path}" 

203 else: 

204 return path 

205 

206 

207class GenericTestCase(_GenericTestCase): 

208 """Test cases for generic manipulation of a `ResourcePath`.""" 

209 

210 def setUp(self) -> None: 

211 if self.scheme is None: 

212 raise unittest.SkipTest("No scheme defined") 

213 self.root = self._make_uri("") 

214 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

215 

216 def test_creation(self) -> None: 

217 self.assertEqual(self.root_uri.scheme, self.scheme) 

218 self.assertEqual(self.root_uri.netloc, self.netloc) 

219 self.assertFalse(self.root_uri.query) 

220 self.assertFalse(self.root_uri.params) 

221 

222 with self.assertRaises(ValueError): 

223 ResourcePath({}) # type: ignore 

224 

225 with self.assertRaises(RuntimeError): 

226 ResourcePath(self.root_uri, isTemporary=True) 

227 

228 file = self.root_uri.join("file.txt", forceDirectory=False) 

229 with self.assertRaises(RuntimeError): 

230 ResourcePath(file, forceDirectory=True) 

231 

232 file = self.root_uri.join("file.txt") 

233 file_as_dir = ResourcePath(file, forceDirectory=True) 

234 self.assertTrue(file_as_dir.isdir()) 

235 

236 dir = self._make_uri("a/b/c/") 

237 with self.assertRaises(ValueError): 

238 ResourcePath(dir, forceDirectory=False) 

239 

240 with self.assertRaises(NotImplementedError): 

241 ResourcePath("unknown://netloc") 

242 

243 replaced = file.replace(fragment="frag") 

244 self.assertEqual(replaced.fragment, "frag") 

245 

246 with self.assertRaises(ValueError): 

247 file.replace(scheme="new") 

248 

249 self.assertNotEqual(replaced, str(replaced)) 

250 self.assertNotEqual(str(replaced), replaced) 

251 

252 def test_extension(self) -> None: 

253 uri = ResourcePath(self._make_uri("dir/test.txt")) 

254 self.assertEqual(uri.updatedExtension(None), uri) 

255 self.assertEqual(uri.updatedExtension(".txt"), uri) 

256 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

257 

258 fits = uri.updatedExtension(".fits.gz") 

259 self.assertEqual(fits.basename(), "test.fits.gz") 

260 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

261 

262 extensionless = self.root_uri.join("no_ext") 

263 self.assertEqual(extensionless.getExtension(), "") 

264 extension = extensionless.updatedExtension(".fits") 

265 self.assertEqual(extension.getExtension(), ".fits") 

266 

267 uri = ResourcePath("test.txt", forceAbsolute=False) 

268 self.assertEqual(uri.getExtension(), ".txt") 

269 uri = ResourcePath(self._make_uri("dir.1/dir.2/test.txt"), forceDirectory=False) 

270 self.assertEqual(uri.getExtension(), ".txt") 

271 uri = ResourcePath(self._make_uri("dir.1/dir.2/"), forceDirectory=True) 

272 self.assertEqual(uri.getExtension(), ".2") 

273 uri = ResourcePath(self._make_uri("dir.1/dir/"), forceDirectory=True) 

274 self.assertEqual(uri.getExtension(), "") 

275 

276 def test_relative(self) -> None: 

277 """Check that we can get subpaths back from two URIs.""" 

278 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

279 self.assertTrue(parent.isdir()) 

280 child = parent.join("dir1/file.txt") 

281 

282 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

283 

284 not_child = ResourcePath("/a/b/dir1/file.txt") 

285 self.assertIsNone(not_child.relative_to(parent)) 

286 self.assertFalse(not_child.isdir()) 

287 

288 not_directory = parent.join("dir1/file2.txt") 

289 self.assertIsNone(child.relative_to(not_directory)) 

290 

291 # Relative URIs 

292 parent = ResourcePath("a/b/", forceAbsolute=False) 

293 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

294 self.assertFalse(child.scheme) 

295 self.assertEqual(child.relative_to(parent), "c/d.txt") 

296 

297 # forceAbsolute=True should work even on an existing ResourcePath 

298 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

299 

300 # Absolute URI and schemeless URI 

301 parent = self.root_uri.join("/a/b/c/") 

302 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

303 

304 # If the child is relative and the parent is absolute we assume 

305 # that the child is a child of the parent unless it uses ".." 

306 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

307 

308 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

309 self.assertIsNone(child.relative_to(parent)) 

310 

311 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

312 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

313 

314 # Test with different netloc 

315 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

316 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True) 

317 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

318 

319 # This is an absolute path so will *always* return a file URI and 

320 # ignore the root parameter. 

321 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True) 

322 self.assertEqual(parent.geturl(), "file:///a/b/c/") 

323 

324 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True) 

325 child = ResourcePath("d/e.txt", root=parent) 

326 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

327 

328 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/"))) 

329 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

330 

331 # Absolute schemeless child with relative parent will always fail. 

332 child = ResourcePath("d/e.txt", root="/a/b/c") 

333 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

334 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

335 

336 def test_parents(self) -> None: 

337 """Test of splitting and parent walking.""" 

338 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

339 child_file = parent.join("subdir/file.txt") 

340 self.assertFalse(child_file.isdir()) 

341 child_subdir, file = child_file.split() 

342 self.assertEqual(file, "file.txt") 

343 self.assertTrue(child_subdir.isdir()) 

344 self.assertEqual(child_file.dirname(), child_subdir) 

345 self.assertEqual(child_file.basename(), file) 

346 self.assertEqual(child_file.parent(), child_subdir) 

347 derived_parent = child_subdir.parent() 

348 self.assertEqual(derived_parent, parent) 

349 self.assertTrue(derived_parent.isdir()) 

350 self.assertEqual(child_file.parent().parent(), parent) 

351 self.assertEqual(child_subdir.dirname(), child_subdir) 

352 

353 def test_escapes(self) -> None: 

354 """Special characters in file paths.""" 

355 src = self.root_uri.join("bbb/???/test.txt") 

356 self.assertNotIn("???", src.path) 

357 self.assertIn("???", src.unquoted_path) 

358 

359 file = src.updatedFile("tests??.txt") 

360 self.assertNotIn("??.txt", file.path) 

361 

362 src = src.updatedFile("tests??.txt") 

363 self.assertIn("??.txt", src.unquoted_path) 

364 

365 # File URI and schemeless URI 

366 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

367 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

368 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

369 

370 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

371 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

372 

373 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

374 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

375 

376 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

377 

378 # dir.join() morphs into a file scheme 

379 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

380 new = dir.join("test_j.txt") 

381 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

382 

383 new2name = "###/test??.txt" 

384 new2 = dir.join(new2name) 

385 self.assertIn("???", new2.unquoted_path) 

386 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

387 

388 fdir = dir.abspath() 

389 self.assertNotIn("???", fdir.path) 

390 self.assertIn("???", fdir.unquoted_path) 

391 self.assertEqual(fdir.scheme, self.scheme) 

392 

393 fnew2 = fdir.join(new2name) 

394 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

395 self.assertNotIn("###", fnew2.path) 

396 

397 # Test that children relative to schemeless and file schemes 

398 # still return the same unquoted name 

399 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

400 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

401 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

402 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

403 

404 # Check for double quoting 

405 plus_path = "/a/b/c+d/" 

406 with self.assertLogs(level="WARNING"): 

407 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

408 self.assertEqual(uri.ospath, plus_path) 

409 

410 # Check that # is not escaped for schemeless URIs 

411 hash_path = "/a/b#/c&d#xyz" 

412 hpos = hash_path.rfind("#") 

413 uri = ResourcePath(hash_path) 

414 self.assertEqual(uri.ospath, hash_path[:hpos]) 

415 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

416 

417 def test_hash(self) -> None: 

418 """Test that we can store URIs in sets and as keys.""" 

419 uri1 = self.root_uri 

420 uri2 = uri1.join("test/") 

421 s = {uri1, uri2} 

422 self.assertIn(uri1, s) 

423 

424 d = {uri1: "1", uri2: "2"} 

425 self.assertEqual(d[uri2], "2") 

426 

427 def test_root_uri(self) -> None: 

428 """Test ResourcePath.root_uri().""" 

429 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

430 self.assertEqual(uri.root_uri().geturl(), self.root) 

431 

432 def test_join(self) -> None: 

433 """Test .join method.""" 

434 root_str = self.root 

435 root = self.root_uri 

436 

437 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

438 add_dir = root.join("b/c/d/") 

439 self.assertTrue(add_dir.isdir()) 

440 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

441 

442 up_relative = root.join("../b/c.txt") 

443 self.assertFalse(up_relative.isdir()) 

444 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

445 

446 quote_example = "hsc/payload/b&c.t@x#t" 

447 needs_quote = root.join(quote_example) 

448 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

449 

450 other = ResourcePath(f"{self.root}test.txt") 

451 self.assertEqual(root.join(other), other) 

452 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}test.txt/b/new.txt") 

453 

454 other = ResourcePath(f"{self.root}text.txt", forceDirectory=False) 

455 with self.assertRaises(ValueError): 

456 other.join("b/new.text") 

457 

458 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

459 ResourcePath("test.qgraph", forceAbsolute=False) 

460 ) 

461 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

462 

463 qgraph = ResourcePath("test.qgraph") # Absolute URI 

464 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph) 

465 self.assertEqual(joined, qgraph) 

466 

467 with self.assertRaises(ValueError): 

468 root.join("dir/", forceDirectory=False) 

469 

470 temp = root.join("dir2/", isTemporary=True) 

471 with self.assertRaises(RuntimeError): 

472 temp.join("test.txt", isTemporary=False) 

473 

474 rel = ResourcePath("new.txt", forceAbsolute=False, forceDirectory=False) 

475 with self.assertRaises(RuntimeError): 

476 root.join(rel, forceDirectory=True) 

477 

478 def test_quoting(self) -> None: 

479 """Check that quoting works.""" 

480 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

481 subpath = "rootdir/dir1+/file?.txt" 

482 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

483 

484 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

485 self.assertEqual(child.basename(), "file?.txt") 

486 self.assertEqual(child.relativeToPathRoot, subpath) 

487 self.assertIn("%", child.path) 

488 self.assertEqual(child.unquoted_path, "/" + subpath) 

489 

490 def test_ordering(self) -> None: 

491 """Check that greater/less comparison operators work.""" 

492 a = self._make_uri("a.txt") 

493 b = self._make_uri("b/") 

494 self.assertLess(a, b) 

495 self.assertFalse(a < a) 

496 self.assertLessEqual(a, b) 

497 self.assertLessEqual(a, a) 

498 self.assertGreater(b, a) 

499 self.assertFalse(b > b) 

500 self.assertGreaterEqual(b, a) 

501 self.assertGreaterEqual(b, b) 

502 

503 

504class GenericReadWriteTestCase(_GenericTestCase): 

505 """Test schemes that can read and write using concrete resources.""" 

506 

507 transfer_modes: tuple[str, ...] = ("copy", "move") 

508 testdir: str | None = None 

509 

510 def setUp(self) -> None: 

511 if self.scheme is None: 

512 raise unittest.SkipTest("No scheme defined") 

513 self.root = self._make_uri("") 

514 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

515 

516 if self.scheme == "file": 

517 # Use a local tempdir because on macOS the temp dirs use symlinks 

518 # so relsymlink gets quite confused. 

519 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir), forceDirectory=True) 

520 else: 

521 # Create random tmp directory relative to the test root. 

522 self.tmpdir = self.root_uri.join( 

523 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)), 

524 forceDirectory=True, 

525 ) 

526 self.tmpdir.mkdir() 

527 

528 def tearDown(self) -> None: 

529 if self.tmpdir and self.tmpdir.isLocal: 

530 removeTestTempDir(self.tmpdir.ospath) 

531 

532 def test_file(self) -> None: 

533 uri = self.tmpdir.join("test.txt") 

534 self.assertFalse(uri.exists(), f"{uri} should not exist") 

535 self.assertTrue(uri.path.endswith("test.txt")) 

536 

537 content = "abcdefghijklmnopqrstuv\n" 

538 uri.write(content.encode()) 

539 self.assertTrue(uri.exists(), f"{uri} should now exist") 

540 self.assertEqual(uri.read().decode(), content) 

541 self.assertEqual(uri.size(), len(content.encode())) 

542 

543 with self.assertRaises(FileExistsError): 

544 uri.write(b"", overwrite=False) 

545 

546 # Not all backends can tell if a remove fails so we can not 

547 # test that a remove of a non-existent entry is guaranteed to raise. 

548 uri.remove() 

549 self.assertFalse(uri.exists()) 

550 

551 # Ideally the test would remove the file again and raise a 

552 # FileNotFoundError. This is not reliable for remote resources 

553 # and doing an explicit check before trying to remove the resource 

554 # just to raise an exception is deemed an unacceptable overhead. 

555 

556 with self.assertRaises(FileNotFoundError): 

557 uri.read() 

558 

559 with self.assertRaises(FileNotFoundError): 

560 self.tmpdir.join("file/not/there.txt").size() 

561 

562 # Check that creating a URI from a URI returns the same thing 

563 uri2 = ResourcePath(uri) 

564 self.assertEqual(uri, uri2) 

565 self.assertEqual(id(uri), id(uri2)) 

566 

567 def test_mkdir(self) -> None: 

568 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

569 newdir.mkdir() 

570 self.assertTrue(newdir.exists()) 

571 self.assertEqual(newdir.size(), 0) 

572 

573 newfile = newdir.join("temp.txt") 

574 newfile.write(b"Data") 

575 self.assertTrue(newfile.exists()) 

576 

577 file = self.tmpdir.join("file.txt") 

578 # Some schemes will realize that the URI is not a file and so 

579 # will raise NotADirectoryError. The file scheme is more permissive 

580 # and lets you write anything but will raise NotADirectoryError 

581 # if a non-directory is already there. We therefore write something 

582 # to the file to ensure that we trigger a portable exception. 

583 file.write(b"") 

584 with self.assertRaises(NotADirectoryError): 

585 file.mkdir() 

586 

587 # The root should exist. 

588 self.root_uri.mkdir() 

589 self.assertTrue(self.root_uri.exists()) 

590 

591 def test_transfer(self) -> None: 

592 src = self.tmpdir.join("test.txt") 

593 content = "Content is some content\nwith something to say\n\n" 

594 src.write(content.encode()) 

595 

596 can_move = "move" in self.transfer_modes 

597 for mode in self.transfer_modes: 

598 if mode == "move": 

599 continue 

600 

601 dest = self.tmpdir.join(f"dest_{mode}.txt") 

602 # Ensure that we get some debugging output. 

603 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

604 dest.transfer_from(src, transfer=mode) 

605 self.assertIn("Transferring ", "\n".join(cm.output)) 

606 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

607 

608 new_content = dest.read().decode() 

609 self.assertEqual(new_content, content) 

610 

611 if mode in ("symlink", "relsymlink"): 

612 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

613 

614 # If the source and destination are hardlinks of each other 

615 # the transfer should work even if overwrite=False. 

616 if mode in ("link", "hardlink"): 

617 dest.transfer_from(src, transfer=mode) 

618 else: 

619 with self.assertRaises( 

620 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

621 ): 

622 dest.transfer_from(src, transfer=mode) 

623 

624 # Transfer again and overwrite. 

625 dest.transfer_from(src, transfer=mode, overwrite=True) 

626 

627 dest.remove() 

628 

629 b = src.read() 

630 self.assertEqual(b.decode(), new_content) 

631 

632 nbytes = 10 

633 subset = src.read(size=nbytes) 

634 self.assertEqual(len(subset), nbytes) 

635 self.assertEqual(subset.decode(), content[:nbytes]) 

636 

637 # Transferring to self should be okay. 

638 src.transfer_from(src, "auto") 

639 

640 with self.assertRaises(ValueError): 

641 src.transfer_from(src, transfer="unknown") 

642 

643 # A move transfer is special. 

644 if can_move: 

645 dest.transfer_from(src, transfer="move") 

646 self.assertFalse(src.exists()) 

647 self.assertTrue(dest.exists()) 

648 else: 

649 src.remove() 

650 

651 dest.remove() 

652 with self.assertRaises(FileNotFoundError): 

653 dest.transfer_from(src, "auto") 

654 

655 def test_local_transfer(self) -> None: 

656 """Test we can transfer to and from local file.""" 

657 remote_src = self.tmpdir.join("src.json") 

658 remote_src.write(b"42") 

659 remote_dest = self.tmpdir.join("dest.json") 

660 

661 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

662 self.assertTrue(tmp.isLocal) 

663 tmp.transfer_from(remote_src, transfer="auto") 

664 self.assertEqual(tmp.read(), remote_src.read()) 

665 

666 remote_dest.transfer_from(tmp, transfer="auto") 

667 self.assertEqual(remote_dest.read(), tmp.read()) 

668 

669 # Temporary (possibly remote) resource. 

670 # Transfers between temporary resources. 

671 with ( 

672 ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp, 

673 ResourcePath.temporary_uri(suffix=".json") as local_tmp, 

674 ): 

675 remote_tmp.write(b"42") 

676 if not remote_tmp.isLocal: 

677 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

678 with self.assertRaises(RuntimeError): 

679 # Trying to symlink a remote resource is not going 

680 # to work. A hardlink could work but would rely 

681 # on the local temp space being on the same 

682 # filesystem as the target. 

683 local_tmp.transfer_from(remote_tmp, transfer) 

684 local_tmp.transfer_from(remote_tmp, "move") 

685 self.assertFalse(remote_tmp.exists()) 

686 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

687 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

688 

689 # Transfer of missing remote. 

690 remote_tmp.remove() 

691 with self.assertRaises(FileNotFoundError): 

692 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

693 

694 def test_local(self) -> None: 

695 """Check that remote resources can be made local.""" 

696 src = self.tmpdir.join("test.txt") 

697 original_content = "Content is some content\nwith something to say\n\n" 

698 src.write(original_content.encode()) 

699 

700 # Run this twice to ensure use of cache in code coverage 

701 # if applicable. 

702 for _ in (1, 2): 

703 with src.as_local() as local_uri: 

704 self.assertTrue(local_uri.isLocal) 

705 content = local_uri.read().decode() 

706 self.assertEqual(content, original_content) 

707 

708 if src.isLocal: 

709 self.assertEqual(src, local_uri) 

710 

711 with self.assertRaises(IsADirectoryError): 

712 with self.root_uri.as_local() as local_uri: 

713 pass 

714 

715 def test_walk(self) -> None: 

716 """Walk a directory hierarchy.""" 

717 root = self.tmpdir.join("walk/") 

718 

719 # Look for a file that is not there 

720 file = root.join("config/basic/butler.yaml") 

721 found_list = list(ResourcePath.findFileResources([file])) 

722 self.assertEqual(found_list[0], file) 

723 

724 # First create the files (content is irrelevant). 

725 expected_files = { 

726 "dir1/a.yaml", 

727 "dir1/b.yaml", 

728 "dir1/c.json", 

729 "dir2/d.json", 

730 "dir2/e.yaml", 

731 } 

732 expected_uris = {root.join(f) for f in expected_files} 

733 for uri in expected_uris: 

734 uri.write(b"") 

735 self.assertTrue(uri.exists()) 

736 

737 # Look for the files. 

738 found = set(ResourcePath.findFileResources([root])) 

739 self.assertEqual(found, expected_uris) 

740 

741 # Now solely the YAML files. 

742 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

743 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

744 self.assertEqual(found, expected_yaml) 

745 

746 # Now two explicit directories and a file 

747 expected = set(expected_yaml) 

748 expected.add(file) 

749 

750 found = set( 

751 ResourcePath.findFileResources( 

752 [file, root.join("dir1/"), root.join("dir2/")], 

753 file_filter=r".*\.yaml$", 

754 ) 

755 ) 

756 self.assertEqual(found, expected) 

757 

758 # Group by directory -- find everything and compare it with what 

759 # we expected to be there in total. 

760 found_yaml = set() 

761 counter = 0 

762 for uris in ResourcePath.findFileResources([file, root], file_filter=r".*\.yaml$", grouped=True): 

763 assert not isinstance(uris, ResourcePath) # for mypy. 

764 found_uris = set(uris) 

765 if found_uris: 

766 counter += 1 

767 

768 found_yaml.update(found_uris) 

769 

770 expected_yaml_2 = expected_yaml 

771 expected_yaml_2.add(file) 

772 self.assertEqual(found_yaml, expected_yaml) 

773 self.assertEqual(counter, 3) 

774 

775 # Grouping but check that single files are returned in a single group 

776 # at the end 

777 file2 = root.join("config/templates/templates-bad.yaml") 

778 found_grouped = [ 

779 list(group) 

780 for group in ResourcePath.findFileResources([file, file2, root.join("dir2/")], grouped=True) 

781 if not isinstance(group, ResourcePath) # For mypy. 

782 ] 

783 self.assertEqual(len(found_grouped), 2, f"Found: {list(found_grouped)}") 

784 self.assertEqual(list(found_grouped[1]), [file, file2]) 

785 

786 with self.assertRaises(ValueError): 

787 # The list forces the generator to run. 

788 list(file.walk()) 

789 

790 # A directory that does not exist returns nothing. 

791 self.assertEqual(list(root.join("dir3/").walk()), []) 

792 

793 def test_large_walk(self) -> None: 

794 # In some systems pagination is used so ensure that we can handle 

795 # large numbers of files. For example S3 limits us to 1000 responses 

796 # per listing call. 

797 created = set() 

798 counter = 1 

799 n_dir1 = 1100 

800 root = self.tmpdir.join("large_walk", forceDirectory=True) 

801 while counter <= n_dir1: 

802 new = ResourcePath(root.join(f"file{counter:04d}.txt")) 

803 new.write(f"{counter}".encode()) 

804 created.add(new) 

805 counter += 1 

806 counter = 1 

807 # Put some in a subdirectory to make sure we are looking in a 

808 # hierarchy. 

809 n_dir2 = 100 

810 subdir = root.join("subdir", forceDirectory=True) 

811 while counter <= n_dir2: 

812 new = ResourcePath(subdir.join(f"file{counter:04d}.txt")) 

813 new.write(f"{counter}".encode()) 

814 created.add(new) 

815 counter += 1 

816 

817 found = set(ResourcePath.findFileResources([root])) 

818 self.assertEqual(len(found), n_dir1 + n_dir2) 

819 self.assertEqual(found, created) 

820 

821 # Again with grouping. 

822 # (mypy gets upset not knowing which of the two options is being 

823 # returned so add useless instance check). 

824 found_list = [ 

825 list(group) 

826 for group in ResourcePath.findFileResources([root], grouped=True) 

827 if not isinstance(group, ResourcePath) # For mypy. 

828 ] 

829 self.assertEqual(len(found_list), 2) 

830 self.assertEqual(len(found_list[0]), n_dir1) 

831 self.assertEqual(len(found_list[1]), n_dir2) 

832 

833 def test_temporary(self) -> None: 

834 prefix = self.tmpdir.join("tmp", forceDirectory=True) 

835 with ResourcePath.temporary_uri(prefix=prefix, suffix=".json") as tmp: 

836 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

837 self.assertTrue(tmp.isabs(), f"uri: {tmp}") 

838 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

839 tmp.write(b"abcd") 

840 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

841 self.assertTrue(tmp.isTemporary) 

842 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

843 

844 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

845 with ResourcePath.temporary_uri(prefix=tmpdir) as tmp: 

846 # Use a specified tmpdir and check it is okay for the file 

847 # to not be created. 

848 self.assertFalse(tmp.getExtension()) 

849 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

850 self.assertEqual(tmp.scheme, self.scheme) 

851 self.assertTrue(tmp.isTemporary) 

852 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists") 

853 

854 # Fake a directory suffix. 

855 with self.assertRaises(NotImplementedError): 

856 with ResourcePath.temporary_uri(prefix=self.root_uri, suffix="xxx/") as tmp: 

857 pass 

858 

859 def test_open(self) -> None: 

860 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

861 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp: 

862 _check_open(self, tmp, mode_suffixes=("", "t")) 

863 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16") 

864 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True) 

865 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True) 

866 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp: 

867 _check_open(self, tmp, mode_suffixes=("b",)) 

868 _check_open(self, tmp, mode_suffixes=("b",), prefer_file_temporary=True) 

869 

870 with self.assertRaises(IsADirectoryError): 

871 with self.root_uri.open(): 

872 pass 

873 

874 def test_mexists(self) -> None: 

875 root = self.tmpdir.join("mexists/") 

876 

877 # A file that is not there. 

878 file = root.join("config/basic/butler.yaml") 

879 

880 # Create some files. 

881 expected_files = { 

882 "dir1/a.yaml", 

883 "dir1/b.yaml", 

884 "dir2/e.yaml", 

885 } 

886 expected_uris = {root.join(f) for f in expected_files} 

887 for uri in expected_uris: 

888 uri.write(b"") 

889 self.assertTrue(uri.exists()) 

890 expected_uris.add(file) 

891 

892 multi = ResourcePath.mexists(expected_uris) 

893 

894 for uri, is_there in multi.items(): 

895 if uri == file: 

896 self.assertFalse(is_there) 

897 else: 

898 self.assertTrue(is_there)