Coverage for python/lsst/resources/tests.py: 9%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import random 

19import string 

20import unittest 

21import urllib.parse 

22import uuid 

23from collections.abc import Callable, Iterable 

24from typing import Any 

25 

26from lsst.resources import ResourcePath 

27from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

28 

29TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

30 

31 

32def _check_open( 

33 test_case: _GenericTestCase | unittest.TestCase, 

34 uri: ResourcePath, 

35 *, 

36 mode_suffixes: Iterable[str] = ("", "t", "b"), 

37 **kwargs: Any, 

38) -> None: 

39 """Test an implementation of ButlerURI.open. 

40 

41 Parameters 

42 ---------- 

43 test_case : `unittest.TestCase` 

44 Test case to use for assertions. 

45 uri : `ResourcePath` 

46 URI to use for tests. Must point to a writeable location that is not 

47 yet occupied by a file. On return, the location may point to a file 

48 only if the test fails. 

49 mode_suffixes : `~collections.abc.Iterable` of `str` 

50 Suffixes to pass as part of the ``mode`` argument to 

51 `ResourcePath.open`, indicating whether to open as binary or as text; 

52 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

53 **kwargs 

54 Additional keyword arguments to forward to all calls to `open`. 

55 """ 

56 text_content = "abcdefghijklmnopqrstuvwxyz🙂" 

57 bytes_content = uuid.uuid4().bytes 

58 content_by_mode_suffix = { 

59 "": text_content, 

60 "t": text_content, 

61 "b": bytes_content, 

62 } 

63 empty_content_by_mode_suffix = { 

64 "": "", 

65 "t": "", 

66 "b": b"", 

67 } 

68 # To appease mypy 

69 double_content_by_mode_suffix = { 

70 "": text_content + text_content, 

71 "t": text_content + text_content, 

72 "b": bytes_content + bytes_content, 

73 } 

74 for mode_suffix in mode_suffixes: 

75 content = content_by_mode_suffix[mode_suffix] 

76 double_content = double_content_by_mode_suffix[mode_suffix] 

77 # Create file with mode='x', which prohibits overwriting. 

78 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

79 write_buffer.write(content) 

80 test_case.assertTrue(uri.exists()) 

81 # Check that opening with 'x' now raises, and does not modify content. 

82 with test_case.assertRaises(FileExistsError): 

83 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

84 write_buffer.write("bad") 

85 # Read the file we created and check the contents. 

86 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

87 test_case.assertEqual(read_buffer.read(), content) 

88 # Check that we can read bytes in a loop and get EOF 

89 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

90 # Seek off the end of the file and should read empty back. 

91 read_buffer.seek(1024) 

92 test_case.assertEqual(read_buffer.tell(), 1024) 

93 content_read = read_buffer.read() # Read as much as we can. 

94 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

95 

96 # First read more than the content. 

97 read_buffer.seek(0) 

98 size = len(content) * 3 

99 chunk_read = read_buffer.read(size) 

100 test_case.assertEqual(chunk_read, content) 

101 

102 # Repeated reads should always return empty string. 

103 chunk_read = read_buffer.read(size) 

104 test_case.assertEqual(len(chunk_read), 0) 

105 chunk_read = read_buffer.read(size) 

106 test_case.assertEqual(len(chunk_read), 0) 

107 

108 # Go back to start of file and read in smaller chunks. 

109 read_buffer.seek(0) 

110 size = len(content) // 3 

111 

112 content_read = empty_content_by_mode_suffix[mode_suffix] 

113 n_reads = 0 

114 while chunk_read := read_buffer.read(size): 

115 content_read += chunk_read 

116 n_reads += 1 

117 if n_reads > 10: # In case EOF never hits because of bug. 

118 raise AssertionError( 

119 f"Failed to stop reading from file after {n_reads} loops. " 

120 f"Read {len(content_read)} bytes/characters. Expected {len(content)}." 

121 ) 

122 test_case.assertEqual(content_read, content) 

123 

124 # Go back to start of file and read the entire thing. 

125 read_buffer.seek(0) 

126 content_read = read_buffer.read() 

127 test_case.assertEqual(content_read, content) 

128 

129 # Seek off the end of the file and should read empty back. 

130 # We run this check twice since in some cases the handle will 

131 # cache knowledge of the file size. 

132 read_buffer.seek(1024) 

133 test_case.assertEqual(read_buffer.tell(), 1024) 

134 content_read = read_buffer.read() 

135 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

136 

137 # Write two copies of the content, overwriting the single copy there. 

138 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

139 write_buffer.write(double_content) 

140 # Read again, this time use mode='r+', which reads what is there and 

141 # then lets us write more; we'll use that to reset the file to one 

142 # copy of the content. 

143 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

144 test_case.assertEqual(rw_buffer.read(), double_content) 

145 rw_buffer.seek(0) 

146 rw_buffer.truncate() 

147 rw_buffer.write(content) 

148 rw_buffer.seek(0) 

149 test_case.assertEqual(rw_buffer.read(), content) 

150 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

151 test_case.assertEqual(read_buffer.read(), content) 

152 # Append some more content to the file; should now have two copies. 

153 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

154 append_buffer.write(content) 

155 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

156 test_case.assertEqual(read_buffer.read(), double_content) 

157 # Final mode to check is w+, which does read/write but truncates first. 

158 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

159 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

160 rw_buffer.write(content) 

161 rw_buffer.seek(0) 

162 test_case.assertEqual(rw_buffer.read(), content) 

163 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

164 test_case.assertEqual(read_buffer.read(), content) 

165 # Remove file to make room for the next loop of tests with this URI. 

166 uri.remove() 

167 

168 

169class _GenericTestCase: 

170 """Generic base class for test mixin.""" 

171 

172 scheme: str | None = None 

173 netloc: str | None = None 

174 base_path: str | None = None 

175 path1 = "test_dir" 

176 path2 = "file.txt" 

177 

178 # Because we use a mixin for tests mypy needs to understand that 

179 # the unittest.TestCase methods exist. 

180 # We do not inherit from unittest.TestCase because that results 

181 # in the tests defined here being run as well as the tests in the 

182 # test file itself. We can make those tests skip but it gives an 

183 # uniformative view of how many tests are running. 

184 assertEqual: Callable 

185 assertNotEqual: Callable 

186 assertIsNone: Callable 

187 assertIn: Callable 

188 assertNotIn: Callable 

189 assertFalse: Callable 

190 assertTrue: Callable 

191 assertRaises: Callable 

192 assertLogs: Callable 

193 

194 def _make_uri(self, path: str, netloc: str | None = None) -> str: 

195 if self.scheme is not None: 

196 if netloc is None: 

197 netloc = self.netloc 

198 if path.startswith("/"): 

199 path = path[1:] 

200 if self.base_path is not None: 

201 path = f"{self.base_path}/{path}".lstrip("/") 

202 

203 return f"{self.scheme}://{netloc}/{path}" 

204 else: 

205 return path 

206 

207 

208class GenericTestCase(_GenericTestCase): 

209 """Test cases for generic manipulation of a `ResourcePath`""" 

210 

211 def setUp(self) -> None: 

212 if self.scheme is None: 

213 raise unittest.SkipTest("No scheme defined") 

214 self.root = self._make_uri("") 

215 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

216 

217 def test_creation(self) -> None: 

218 self.assertEqual(self.root_uri.scheme, self.scheme) 

219 self.assertEqual(self.root_uri.netloc, self.netloc) 

220 self.assertFalse(self.root_uri.query) 

221 self.assertFalse(self.root_uri.params) 

222 

223 with self.assertRaises(ValueError): 

224 ResourcePath({}) # type: ignore 

225 

226 with self.assertRaises(RuntimeError): 

227 ResourcePath(self.root_uri, isTemporary=True) 

228 

229 file = self.root_uri.join("file.txt") 

230 with self.assertRaises(RuntimeError): 

231 ResourcePath(file, forceDirectory=True) 

232 

233 with self.assertRaises(NotImplementedError): 

234 ResourcePath("unknown://netloc") 

235 

236 replaced = file.replace(fragment="frag") 

237 self.assertEqual(replaced.fragment, "frag") 

238 

239 with self.assertRaises(ValueError): 

240 file.replace(scheme="new") 

241 

242 self.assertNotEqual(replaced, str(replaced)) 

243 self.assertNotEqual(str(replaced), replaced) 

244 

245 def test_extension(self) -> None: 

246 uri = ResourcePath(self._make_uri("dir/test.txt")) 

247 self.assertEqual(uri.updatedExtension(None), uri) 

248 self.assertEqual(uri.updatedExtension(".txt"), uri) 

249 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

250 

251 fits = uri.updatedExtension(".fits.gz") 

252 self.assertEqual(fits.basename(), "test.fits.gz") 

253 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

254 

255 extensionless = self.root_uri.join("no_ext") 

256 self.assertEqual(extensionless.getExtension(), "") 

257 extension = extensionless.updatedExtension(".fits") 

258 self.assertEqual(extension.getExtension(), ".fits") 

259 

260 def test_relative(self) -> None: 

261 """Check that we can get subpaths back from two URIs""" 

262 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

263 self.assertTrue(parent.isdir()) 

264 child = parent.join("dir1/file.txt") 

265 

266 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

267 

268 not_child = ResourcePath("/a/b/dir1/file.txt") 

269 self.assertIsNone(not_child.relative_to(parent)) 

270 self.assertFalse(not_child.isdir()) 

271 

272 not_directory = parent.join("dir1/file2.txt") 

273 self.assertIsNone(child.relative_to(not_directory)) 

274 

275 # Relative URIs 

276 parent = ResourcePath("a/b/", forceAbsolute=False) 

277 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

278 self.assertFalse(child.scheme) 

279 self.assertEqual(child.relative_to(parent), "c/d.txt") 

280 

281 # forceAbsolute=True should work even on an existing ResourcePath 

282 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

283 

284 # Absolute URI and schemeless URI 

285 parent = self.root_uri.join("/a/b/c/") 

286 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

287 

288 # If the child is relative and the parent is absolute we assume 

289 # that the child is a child of the parent unless it uses ".." 

290 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

291 

292 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

293 self.assertIsNone(child.relative_to(parent)) 

294 

295 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

296 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

297 

298 # Test with different netloc 

299 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

300 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True) 

301 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

302 

303 # This is an absolute path so will *always* return a file URI and 

304 # ignore the root parameter. 

305 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True) 

306 self.assertEqual(parent.geturl(), "file:///a/b/c/") 

307 

308 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True) 

309 child = ResourcePath("d/e.txt", root=parent) 

310 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

311 

312 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/"))) 

313 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

314 

315 # Absolute schemeless child with relative parent will always fail. 

316 child = ResourcePath("d/e.txt", root="/a/b/c") 

317 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

318 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

319 

320 def test_parents(self) -> None: 

321 """Test of splitting and parent walking.""" 

322 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

323 child_file = parent.join("subdir/file.txt") 

324 self.assertFalse(child_file.isdir()) 

325 child_subdir, file = child_file.split() 

326 self.assertEqual(file, "file.txt") 

327 self.assertTrue(child_subdir.isdir()) 

328 self.assertEqual(child_file.dirname(), child_subdir) 

329 self.assertEqual(child_file.basename(), file) 

330 self.assertEqual(child_file.parent(), child_subdir) 

331 derived_parent = child_subdir.parent() 

332 self.assertEqual(derived_parent, parent) 

333 self.assertTrue(derived_parent.isdir()) 

334 self.assertEqual(child_file.parent().parent(), parent) 

335 

336 def test_escapes(self) -> None: 

337 """Special characters in file paths""" 

338 src = self.root_uri.join("bbb/???/test.txt") 

339 self.assertNotIn("???", src.path) 

340 self.assertIn("???", src.unquoted_path) 

341 

342 file = src.updatedFile("tests??.txt") 

343 self.assertNotIn("??.txt", file.path) 

344 

345 src = src.updatedFile("tests??.txt") 

346 self.assertIn("??.txt", src.unquoted_path) 

347 

348 # File URI and schemeless URI 

349 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

350 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

351 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

352 

353 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

354 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

355 

356 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

357 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

358 

359 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

360 

361 # dir.join() morphs into a file scheme 

362 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

363 new = dir.join("test_j.txt") 

364 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

365 

366 new2name = "###/test??.txt" 

367 new2 = dir.join(new2name) 

368 self.assertIn("???", new2.unquoted_path) 

369 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

370 

371 fdir = dir.abspath() 

372 self.assertNotIn("???", fdir.path) 

373 self.assertIn("???", fdir.unquoted_path) 

374 self.assertEqual(fdir.scheme, self.scheme) 

375 

376 fnew2 = fdir.join(new2name) 

377 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

378 self.assertNotIn("###", fnew2.path) 

379 

380 # Test that children relative to schemeless and file schemes 

381 # still return the same unquoted name 

382 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

383 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

384 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

385 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

386 

387 # Check for double quoting 

388 plus_path = "/a/b/c+d/" 

389 with self.assertLogs(level="WARNING"): 

390 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

391 self.assertEqual(uri.ospath, plus_path) 

392 

393 # Check that # is not escaped for schemeless URIs 

394 hash_path = "/a/b#/c&d#xyz" 

395 hpos = hash_path.rfind("#") 

396 uri = ResourcePath(hash_path) 

397 self.assertEqual(uri.ospath, hash_path[:hpos]) 

398 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

399 

400 def test_hash(self) -> None: 

401 """Test that we can store URIs in sets and as keys.""" 

402 uri1 = self.root_uri 

403 uri2 = uri1.join("test/") 

404 s = {uri1, uri2} 

405 self.assertIn(uri1, s) 

406 

407 d = {uri1: "1", uri2: "2"} 

408 self.assertEqual(d[uri2], "2") 

409 

410 def test_root_uri(self) -> None: 

411 """Test ResourcePath.root_uri().""" 

412 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

413 self.assertEqual(uri.root_uri().geturl(), self.root) 

414 

415 def test_join(self) -> None: 

416 """Test .join method.""" 

417 root_str = self.root 

418 root = self.root_uri 

419 

420 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

421 add_dir = root.join("b/c/d/") 

422 self.assertTrue(add_dir.isdir()) 

423 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

424 

425 up_relative = root.join("../b/c.txt") 

426 self.assertFalse(up_relative.isdir()) 

427 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

428 

429 quote_example = "hsc/payload/b&c.t@x#t" 

430 needs_quote = root.join(quote_example) 

431 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

432 

433 other = ResourcePath(f"{self.root}test.txt") 

434 self.assertEqual(root.join(other), other) 

435 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

436 

437 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

438 ResourcePath("test.qgraph", forceAbsolute=False) 

439 ) 

440 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

441 

442 qgraph = ResourcePath("test.qgraph") # Absolute URI 

443 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph) 

444 self.assertEqual(joined, qgraph) 

445 

446 def test_quoting(self) -> None: 

447 """Check that quoting works.""" 

448 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

449 subpath = "rootdir/dir1+/file?.txt" 

450 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

451 

452 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

453 self.assertEqual(child.basename(), "file?.txt") 

454 self.assertEqual(child.relativeToPathRoot, subpath) 

455 self.assertIn("%", child.path) 

456 self.assertEqual(child.unquoted_path, "/" + subpath) 

457 

458 def test_ordering(self) -> None: 

459 """Check that greater/less comparison operators work.""" 

460 a = self._make_uri("a.txt") 

461 b = self._make_uri("b/") 

462 self.assertTrue(a < b) 

463 self.assertFalse(a < a) 

464 self.assertTrue(a <= b) 

465 self.assertTrue(a <= a) 

466 self.assertTrue(b > a) 

467 self.assertFalse(b > b) 

468 self.assertTrue(b >= a) 

469 self.assertTrue(b >= b) 

470 

471 

472class GenericReadWriteTestCase(_GenericTestCase): 

473 """Test schemes that can read and write using concrete resources.""" 

474 

475 transfer_modes: tuple[str, ...] = ("copy", "move") 

476 testdir: str | None = None 

477 

478 def setUp(self) -> None: 

479 if self.scheme is None: 

480 raise unittest.SkipTest("No scheme defined") 

481 self.root = self._make_uri("") 

482 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

483 

484 if self.scheme == "file": 

485 # Use a local tempdir because on macOS the temp dirs use symlinks 

486 # so relsymlink gets quite confused. 

487 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

488 else: 

489 # Create random tmp directory relative to the test root. 

490 self.tmpdir = self.root_uri.join( 

491 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)), 

492 forceDirectory=True, 

493 ) 

494 self.tmpdir.mkdir() 

495 

496 def tearDown(self) -> None: 

497 if self.tmpdir: 

498 if self.tmpdir.isLocal: 

499 removeTestTempDir(self.tmpdir.ospath) 

500 

501 def test_file(self) -> None: 

502 uri = self.tmpdir.join("test.txt") 

503 self.assertFalse(uri.exists(), f"{uri} should not exist") 

504 self.assertTrue(uri.path.endswith("test.txt")) 

505 

506 content = "abcdefghijklmnopqrstuv\n" 

507 uri.write(content.encode()) 

508 self.assertTrue(uri.exists(), f"{uri} should now exist") 

509 self.assertEqual(uri.read().decode(), content) 

510 self.assertEqual(uri.size(), len(content.encode())) 

511 

512 with self.assertRaises(FileExistsError): 

513 uri.write(b"", overwrite=False) 

514 

515 # Not all backends can tell if a remove fails so we can not 

516 # test that a remove of a non-existent entry is guaranteed to raise. 

517 uri.remove() 

518 self.assertFalse(uri.exists()) 

519 

520 # Ideally the test would remove the file again and raise a 

521 # FileNotFoundError. This is not reliable for remote resources 

522 # and doing an explicit check before trying to remove the resource 

523 # just to raise an exception is deemed an unacceptable overhead. 

524 

525 with self.assertRaises(FileNotFoundError): 

526 uri.read() 

527 

528 with self.assertRaises(FileNotFoundError): 

529 self.tmpdir.join("file/not/there.txt").size() 

530 

531 # Check that creating a URI from a URI returns the same thing 

532 uri2 = ResourcePath(uri) 

533 self.assertEqual(uri, uri2) 

534 self.assertEqual(id(uri), id(uri2)) 

535 

536 def test_mkdir(self) -> None: 

537 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

538 newdir.mkdir() 

539 self.assertTrue(newdir.exists()) 

540 self.assertEqual(newdir.size(), 0) 

541 

542 newfile = newdir.join("temp.txt") 

543 newfile.write(b"Data") 

544 self.assertTrue(newfile.exists()) 

545 

546 file = self.tmpdir.join("file.txt") 

547 # Some schemes will realize that the URI is not a file and so 

548 # will raise NotADirectoryError. The file scheme is more permissive 

549 # and lets you write anything but will raise NotADirectoryError 

550 # if a non-directory is already there. We therefore write something 

551 # to the file to ensure that we trigger a portable exception. 

552 file.write(b"") 

553 with self.assertRaises(NotADirectoryError): 

554 file.mkdir() 

555 

556 # The root should exist. 

557 self.root_uri.mkdir() 

558 self.assertTrue(self.root_uri.exists()) 

559 

560 def test_transfer(self) -> None: 

561 src = self.tmpdir.join("test.txt") 

562 content = "Content is some content\nwith something to say\n\n" 

563 src.write(content.encode()) 

564 

565 can_move = "move" in self.transfer_modes 

566 for mode in self.transfer_modes: 

567 if mode == "move": 

568 continue 

569 

570 dest = self.tmpdir.join(f"dest_{mode}.txt") 

571 # Ensure that we get some debugging output. 

572 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

573 dest.transfer_from(src, transfer=mode) 

574 self.assertIn("Transferring ", "\n".join(cm.output)) 

575 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

576 

577 new_content = dest.read().decode() 

578 self.assertEqual(new_content, content) 

579 

580 if mode in ("symlink", "relsymlink"): 

581 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

582 

583 # If the source and destination are hardlinks of each other 

584 # the transfer should work even if overwrite=False. 

585 if mode in ("link", "hardlink"): 

586 dest.transfer_from(src, transfer=mode) 

587 else: 

588 with self.assertRaises( 

589 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

590 ): 

591 dest.transfer_from(src, transfer=mode) 

592 

593 # Transfer again and overwrite. 

594 dest.transfer_from(src, transfer=mode, overwrite=True) 

595 

596 dest.remove() 

597 

598 b = src.read() 

599 self.assertEqual(b.decode(), new_content) 

600 

601 nbytes = 10 

602 subset = src.read(size=nbytes) 

603 self.assertEqual(len(subset), nbytes) 

604 self.assertEqual(subset.decode(), content[:nbytes]) 

605 

606 # Transferring to self should be okay. 

607 src.transfer_from(src, "auto") 

608 

609 with self.assertRaises(ValueError): 

610 src.transfer_from(src, transfer="unknown") 

611 

612 # A move transfer is special. 

613 if can_move: 

614 dest.transfer_from(src, transfer="move") 

615 self.assertFalse(src.exists()) 

616 self.assertTrue(dest.exists()) 

617 else: 

618 src.remove() 

619 

620 dest.remove() 

621 with self.assertRaises(FileNotFoundError): 

622 dest.transfer_from(src, "auto") 

623 

624 def test_local_transfer(self) -> None: 

625 """Test we can transfer to and from local file.""" 

626 remote_src = self.tmpdir.join("src.json") 

627 remote_src.write(b"42") 

628 remote_dest = self.tmpdir.join("dest.json") 

629 

630 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

631 self.assertTrue(tmp.isLocal) 

632 tmp.transfer_from(remote_src, transfer="auto") 

633 self.assertEqual(tmp.read(), remote_src.read()) 

634 

635 remote_dest.transfer_from(tmp, transfer="auto") 

636 self.assertEqual(remote_dest.read(), tmp.read()) 

637 

638 # Temporary (possibly remote) resource. 

639 # Transfers between temporary resources. 

640 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

641 # Temporary local resource. 

642 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

643 remote_tmp.write(b"42") 

644 if not remote_tmp.isLocal: 

645 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

646 with self.assertRaises(RuntimeError): 

647 # Trying to symlink a remote resource is not going 

648 # to work. A hardlink could work but would rely 

649 # on the local temp space being on the same 

650 # filesystem as the target. 

651 local_tmp.transfer_from(remote_tmp, transfer) 

652 local_tmp.transfer_from(remote_tmp, "move") 

653 self.assertFalse(remote_tmp.exists()) 

654 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

655 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

656 

657 # Transfer of missing remote. 

658 remote_tmp.remove() 

659 with self.assertRaises(FileNotFoundError): 

660 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

661 

662 def test_local(self) -> None: 

663 """Check that remote resources can be made local.""" 

664 src = self.tmpdir.join("test.txt") 

665 original_content = "Content is some content\nwith something to say\n\n" 

666 src.write(original_content.encode()) 

667 

668 # Run this twice to ensure use of cache in code coverage 

669 # if applicable. 

670 for _ in (1, 2): 

671 with src.as_local() as local_uri: 

672 self.assertTrue(local_uri.isLocal) 

673 content = local_uri.read().decode() 

674 self.assertEqual(content, original_content) 

675 

676 if src.isLocal: 

677 self.assertEqual(src, local_uri) 

678 

679 with self.assertRaises(IsADirectoryError): 

680 with self.root_uri.as_local() as local_uri: 

681 pass 

682 

683 def test_walk(self) -> None: 

684 """Walk a directory hierarchy.""" 

685 root = self.tmpdir.join("walk/") 

686 

687 # Look for a file that is not there 

688 file = root.join("config/basic/butler.yaml") 

689 found_list = list(ResourcePath.findFileResources([file])) 

690 self.assertEqual(found_list[0], file) 

691 

692 # First create the files (content is irrelevant). 

693 expected_files = { 

694 "dir1/a.yaml", 

695 "dir1/b.yaml", 

696 "dir1/c.json", 

697 "dir2/d.json", 

698 "dir2/e.yaml", 

699 } 

700 expected_uris = {root.join(f) for f in expected_files} 

701 for uri in expected_uris: 

702 uri.write(b"") 

703 self.assertTrue(uri.exists()) 

704 

705 # Look for the files. 

706 found = set(ResourcePath.findFileResources([root])) 

707 self.assertEqual(found, expected_uris) 

708 

709 # Now solely the YAML files. 

710 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

711 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

712 self.assertEqual(found, expected_yaml) 

713 

714 # Now two explicit directories and a file 

715 expected = set(expected_yaml) 

716 expected.add(file) 

717 

718 found = set( 

719 ResourcePath.findFileResources( 

720 [file, root.join("dir1/"), root.join("dir2/")], 

721 file_filter=r".*\.yaml$", 

722 ) 

723 ) 

724 self.assertEqual(found, expected) 

725 

726 # Group by directory -- find everything and compare it with what