Coverage for python/lsst/resources/tests.py: 9%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11from __future__ import annotations 

12 

13__all__ = ["GenericTestCase", "GenericReadWriteTestCase"] 

14 

15import logging 

16import os 

17import pathlib 

18import random 

19import string 

20import unittest 

21import urllib.parse 

22import uuid 

23from typing import Any, Callable, Iterable, Optional, Union 

24 

25from lsst.resources import ResourcePath 

26from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

27 

28TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

29 

30 

31def _check_open( 

32 test_case: Union[_GenericTestCase, unittest.TestCase], 

33 uri: ResourcePath, 

34 *, 

35 mode_suffixes: Iterable[str] = ("", "t", "b"), 

36 **kwargs: Any, 

37) -> None: 

38 """Test an implementation of ButlerURI.open. 

39 

40 Parameters 

41 ---------- 

42 test_case : `unittest.TestCase` 

43 Test case to use for assertions. 

44 uri : `ResourcePath` 

45 URI to use for tests. Must point to a writeable location that is not 

46 yet occupied by a file. On return, the location may point to a file 

47 only if the test fails. 

48 mode_suffixes : `Iterable` of `str` 

49 Suffixes to pass as part of the ``mode`` argument to 

50 `ResourcePath.open`, indicating whether to open as binary or as text; 

51 the only permitted elements are ``""``, ``"t"``, and ``"b"`. 

52 **kwargs 

53 Additional keyword arguments to forward to all calls to `open`. 

54 """ 

55 text_content = "abcdefghijklmnopqrstuvwxyz🙂" 

56 bytes_content = uuid.uuid4().bytes 

57 content_by_mode_suffix = { 

58 "": text_content, 

59 "t": text_content, 

60 "b": bytes_content, 

61 } 

62 empty_content_by_mode_suffix = { 

63 "": "", 

64 "t": "", 

65 "b": b"", 

66 } 

67 # To appease mypy 

68 double_content_by_mode_suffix = { 

69 "": text_content + text_content, 

70 "t": text_content + text_content, 

71 "b": bytes_content + bytes_content, 

72 } 

73 for mode_suffix in mode_suffixes: 

74 content = content_by_mode_suffix[mode_suffix] 

75 double_content = double_content_by_mode_suffix[mode_suffix] 

76 # Create file with mode='x', which prohibits overwriting. 

77 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

78 write_buffer.write(content) 

79 test_case.assertTrue(uri.exists()) 

80 # Check that opening with 'x' now raises, and does not modify content. 

81 with test_case.assertRaises(FileExistsError): 

82 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

83 write_buffer.write("bad") 

84 # Read the file we created and check the contents. 

85 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

86 test_case.assertEqual(read_buffer.read(), content) 

87 # Check that we can read bytes in a loop and get EOF 

88 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

89 # Seek off the end of the file and should read empty back. 

90 read_buffer.seek(1024) 

91 test_case.assertEqual(read_buffer.tell(), 1024) 

92 content_read = read_buffer.read() # Read as much as we can. 

93 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

94 

95 # First read more than the content. 

96 read_buffer.seek(0) 

97 size = len(content) * 3 

98 chunk_read = read_buffer.read(size) 

99 test_case.assertEqual(chunk_read, content) 

100 

101 # Repeated reads should always return empty string. 

102 chunk_read = read_buffer.read(size) 

103 test_case.assertEqual(len(chunk_read), 0) 

104 chunk_read = read_buffer.read(size) 

105 test_case.assertEqual(len(chunk_read), 0) 

106 

107 # Go back to start of file and read in smaller chunks. 

108 read_buffer.seek(0) 

109 size = len(content) // 3 

110 

111 content_read = empty_content_by_mode_suffix[mode_suffix] 

112 n_reads = 0 

113 while chunk_read := read_buffer.read(size): 

114 content_read += chunk_read 

115 n_reads += 1 

116 if n_reads > 10: # In case EOF never hits because of bug. 

117 raise AssertionError( 

118 f"Failed to stop reading from file after {n_reads} loops. " 

119 f"Read {len(content_read)} bytes/characters. Expected {len(content)}." 

120 ) 

121 test_case.assertEqual(content_read, content) 

122 

123 # Go back to start of file and read the entire thing. 

124 read_buffer.seek(0) 

125 content_read = read_buffer.read() 

126 test_case.assertEqual(content_read, content) 

127 

128 # Seek off the end of the file and should read empty back. 

129 # We run this check twice since in some cases the handle will 

130 # cache knowledge of the file size. 

131 read_buffer.seek(1024) 

132 test_case.assertEqual(read_buffer.tell(), 1024) 

133 content_read = read_buffer.read() 

134 test_case.assertEqual(len(content_read), 0, f"Read: {content_read!r}, expected empty.") 

135 

136 # Write two copies of the content, overwriting the single copy there. 

137 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

138 write_buffer.write(double_content) 

139 # Read again, this time use mode='r+', which reads what is there and 

140 # then lets us write more; we'll use that to reset the file to one 

141 # copy of the content. 

142 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

143 test_case.assertEqual(rw_buffer.read(), double_content) 

144 rw_buffer.seek(0) 

145 rw_buffer.truncate() 

146 rw_buffer.write(content) 

147 rw_buffer.seek(0) 

148 test_case.assertEqual(rw_buffer.read(), content) 

149 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

150 test_case.assertEqual(read_buffer.read(), content) 

151 # Append some more content to the file; should now have two copies. 

152 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

153 append_buffer.write(content) 

154 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

155 test_case.assertEqual(read_buffer.read(), double_content) 

156 # Final mode to check is w+, which does read/write but truncates first. 

157 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

158 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

159 rw_buffer.write(content) 

160 rw_buffer.seek(0) 

161 test_case.assertEqual(rw_buffer.read(), content) 

162 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

163 test_case.assertEqual(read_buffer.read(), content) 

164 # Remove file to make room for the next loop of tests with this URI. 

165 uri.remove() 

166 

167 

168class _GenericTestCase: 

169 """Generic base class for test mixin.""" 

170 

171 scheme: Optional[str] = None 

172 netloc: Optional[str] = None 

173 base_path: Optional[str] = None 

174 path1 = "test_dir" 

175 path2 = "file.txt" 

176 

177 # Because we use a mixin for tests mypy needs to understand that 

178 # the unittest.TestCase methods exist. 

179 # We do not inherit from unittest.TestCase because that results 

180 # in the tests defined here being run as well as the tests in the 

181 # test file itself. We can make those tests skip but it gives an 

182 # uniformative view of how many tests are running. 

183 assertEqual: Callable 

184 assertNotEqual: Callable 

185 assertIsNone: Callable 

186 assertIn: Callable 

187 assertNotIn: Callable 

188 assertFalse: Callable 

189 assertTrue: Callable 

190 assertRaises: Callable 

191 assertLogs: Callable 

192 

193 def _make_uri(self, path: str, netloc: Optional[str] = None) -> str: 

194 if self.scheme is not None: 

195 if netloc is None: 

196 netloc = self.netloc 

197 if path.startswith("/"): 

198 path = path[1:] 

199 if self.base_path is not None: 

200 path = f"{self.base_path}/{path}".lstrip("/") 

201 

202 return f"{self.scheme}://{netloc}/{path}" 

203 else: 

204 return path 

205 

206 

207class GenericTestCase(_GenericTestCase): 

208 """Test cases for generic manipulation of a `ResourcePath`""" 

209 

210 def setUp(self) -> None: 

211 if self.scheme is None: 

212 raise unittest.SkipTest("No scheme defined") 

213 self.root = self._make_uri("") 

214 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

215 

216 def test_creation(self) -> None: 

217 self.assertEqual(self.root_uri.scheme, self.scheme) 

218 self.assertEqual(self.root_uri.netloc, self.netloc) 

219 self.assertFalse(self.root_uri.query) 

220 self.assertFalse(self.root_uri.params) 

221 

222 with self.assertRaises(ValueError): 

223 ResourcePath({}) # type: ignore 

224 

225 with self.assertRaises(RuntimeError): 

226 ResourcePath(self.root_uri, isTemporary=True) 

227 

228 file = self.root_uri.join("file.txt") 

229 with self.assertRaises(RuntimeError): 

230 ResourcePath(file, forceDirectory=True) 

231 

232 with self.assertRaises(NotImplementedError): 

233 ResourcePath("unknown://netloc") 

234 

235 replaced = file.replace(fragment="frag") 

236 self.assertEqual(replaced.fragment, "frag") 

237 

238 with self.assertRaises(ValueError): 

239 file.replace(scheme="new") 

240 

241 self.assertNotEqual(replaced, str(replaced)) 

242 self.assertNotEqual(str(replaced), replaced) 

243 

244 def test_extension(self) -> None: 

245 uri = ResourcePath(self._make_uri("dir/test.txt")) 

246 self.assertEqual(uri.updatedExtension(None), uri) 

247 self.assertEqual(uri.updatedExtension(".txt"), uri) 

248 self.assertEqual(id(uri.updatedExtension(".txt")), id(uri)) 

249 

250 fits = uri.updatedExtension(".fits.gz") 

251 self.assertEqual(fits.basename(), "test.fits.gz") 

252 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

253 

254 extensionless = self.root_uri.join("no_ext") 

255 self.assertEqual(extensionless.getExtension(), "") 

256 extension = extensionless.updatedExtension(".fits") 

257 self.assertEqual(extension.getExtension(), ".fits") 

258 

259 def test_relative(self) -> None: 

260 """Check that we can get subpaths back from two URIs""" 

261 parent = ResourcePath(self._make_uri(self.path1), forceDirectory=True) 

262 self.assertTrue(parent.isdir()) 

263 child = parent.join("dir1/file.txt") 

264 

265 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

266 

267 not_child = ResourcePath("/a/b/dir1/file.txt") 

268 self.assertIsNone(not_child.relative_to(parent)) 

269 self.assertFalse(not_child.isdir()) 

270 

271 not_directory = parent.join("dir1/file2.txt") 

272 self.assertIsNone(child.relative_to(not_directory)) 

273 

274 # Relative URIs 

275 parent = ResourcePath("a/b/", forceAbsolute=False) 

276 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

277 self.assertFalse(child.scheme) 

278 self.assertEqual(child.relative_to(parent), "c/d.txt") 

279 

280 # forceAbsolute=True should work even on an existing ResourcePath 

281 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

282 

283 # Absolute URI and schemeless URI 

284 parent = self.root_uri.join("/a/b/c/") 

285 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

286 

287 # If the child is relative and the parent is absolute we assume 

288 # that the child is a child of the parent unless it uses ".." 

289 self.assertEqual(child.relative_to(parent), "e/f/g.txt", f"{child}.relative_to({parent})") 

290 

291 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

292 self.assertIsNone(child.relative_to(parent)) 

293 

294 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

295 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

296 

297 # Test with different netloc 

298 child = ResourcePath(self._make_uri("a/b/c.txt", netloc="my.host")) 

299 parent = ResourcePath(self._make_uri("a", netloc="other"), forceDirectory=True) 

300 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

301 

302 # This is an absolute path so will *always* return a file URI and 

303 # ignore the root parameter. 

304 parent = ResourcePath("/a/b/c", root=self.root_uri, forceDirectory=True) 

305 self.assertEqual(parent.geturl(), "file:///a/b/c/") 

306 

307 parent = ResourcePath(self._make_uri("/a/b/c"), forceDirectory=True) 

308 child = ResourcePath("d/e.txt", root=parent) 

309 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

310 

311 parent = ResourcePath("c/", root=ResourcePath(self._make_uri("/a/b/"))) 

312 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

313 

314 # Absolute schemeless child with relative parent will always fail. 

315 child = ResourcePath("d/e.txt", root="/a/b/c") 

316 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

317 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

318 

319 def test_parents(self) -> None: 

320 """Test of splitting and parent walking.""" 

321 parent = ResourcePath(self._make_uri("somedir"), forceDirectory=True) 

322 child_file = parent.join("subdir/file.txt") 

323 self.assertFalse(child_file.isdir()) 

324 child_subdir, file = child_file.split() 

325 self.assertEqual(file, "file.txt") 

326 self.assertTrue(child_subdir.isdir()) 

327 self.assertEqual(child_file.dirname(), child_subdir) 

328 self.assertEqual(child_file.basename(), file) 

329 self.assertEqual(child_file.parent(), child_subdir) 

330 derived_parent = child_subdir.parent() 

331 self.assertEqual(derived_parent, parent) 

332 self.assertTrue(derived_parent.isdir()) 

333 self.assertEqual(child_file.parent().parent(), parent) 

334 

335 def test_escapes(self) -> None: 

336 """Special characters in file paths""" 

337 src = self.root_uri.join("bbb/???/test.txt") 

338 self.assertNotIn("???", src.path) 

339 self.assertIn("???", src.unquoted_path) 

340 

341 file = src.updatedFile("tests??.txt") 

342 self.assertNotIn("??.txt", file.path) 

343 

344 src = src.updatedFile("tests??.txt") 

345 self.assertIn("??.txt", src.unquoted_path) 

346 

347 # File URI and schemeless URI 

348 parent = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/"))) 

349 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

350 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

351 

352 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

353 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

354 

355 child = ResourcePath(self._make_uri(urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt"))) 

356 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

357 

358 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

359 

360 # dir.join() morphs into a file scheme 

361 dir = ResourcePath(self._make_uri(urllib.parse.quote("bbb/???/"))) 

362 new = dir.join("test_j.txt") 

363 self.assertIn("???", new.unquoted_path, f"Checking {new}") 

364 

365 new2name = "###/test??.txt" 

366 new2 = dir.join(new2name) 

367 self.assertIn("???", new2.unquoted_path) 

368 self.assertTrue(new2.unquoted_path.endswith(new2name)) 

369 

370 fdir = dir.abspath() 

371 self.assertNotIn("???", fdir.path) 

372 self.assertIn("???", fdir.unquoted_path) 

373 self.assertEqual(fdir.scheme, self.scheme) 

374 

375 fnew2 = fdir.join(new2name) 

376 self.assertTrue(fnew2.unquoted_path.endswith(new2name)) 

377 self.assertNotIn("###", fnew2.path) 

378 

379 # Test that children relative to schemeless and file schemes 

380 # still return the same unquoted name 

381 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

382 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

383 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

384 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

385 

386 # Check for double quoting 

387 plus_path = "/a/b/c+d/" 

388 with self.assertLogs(level="WARNING"): 

389 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

390 self.assertEqual(uri.ospath, plus_path) 

391 

392 # Check that # is not escaped for schemeless URIs 

393 hash_path = "/a/b#/c&d#xyz" 

394 hpos = hash_path.rfind("#") 

395 uri = ResourcePath(hash_path) 

396 self.assertEqual(uri.ospath, hash_path[:hpos]) 

397 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

398 

399 def test_hash(self) -> None: 

400 """Test that we can store URIs in sets and as keys.""" 

401 uri1 = self.root_uri 

402 uri2 = uri1.join("test/") 

403 s = {uri1, uri2} 

404 self.assertIn(uri1, s) 

405 

406 d = {uri1: "1", uri2: "2"} 

407 self.assertEqual(d[uri2], "2") 

408 

409 def test_root_uri(self) -> None: 

410 """Test ResourcePath.root_uri().""" 

411 uri = ResourcePath(self._make_uri("a/b/c.txt")) 

412 self.assertEqual(uri.root_uri().geturl(), self.root) 

413 

414 def test_join(self) -> None: 

415 """Test .join method.""" 

416 root_str = self.root 

417 root = self.root_uri 

418 

419 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

420 add_dir = root.join("b/c/d/") 

421 self.assertTrue(add_dir.isdir()) 

422 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

423 

424 up_relative = root.join("../b/c.txt") 

425 self.assertFalse(up_relative.isdir()) 

426 self.assertEqual(up_relative.geturl(), f"{root_str}b/c.txt") 

427 

428 quote_example = "hsc/payload/b&c.t@x#t" 

429 needs_quote = root.join(quote_example) 

430 self.assertEqual(needs_quote.unquoted_path, "/" + quote_example) 

431 

432 other = ResourcePath(f"{self.root}test.txt") 

433 self.assertEqual(root.join(other), other) 

434 self.assertEqual(other.join("b/new.txt").geturl(), f"{self.root}b/new.txt") 

435 

436 joined = ResourcePath(f"{self.root}hsc/payload/").join( 

437 ResourcePath("test.qgraph", forceAbsolute=False) 

438 ) 

439 self.assertEqual(joined, ResourcePath(f"{self.root}hsc/payload/test.qgraph")) 

440 

441 qgraph = ResourcePath("test.qgraph") # Absolute URI 

442 joined = ResourcePath(f"{self.root}hsc/payload/").join(qgraph) 

443 self.assertEqual(joined, qgraph) 

444 

445 def test_quoting(self) -> None: 

446 """Check that quoting works.""" 

447 parent = ResourcePath(self._make_uri("rootdir"), forceDirectory=True) 

448 subpath = "rootdir/dir1+/file?.txt" 

449 child = ResourcePath(self._make_uri(urllib.parse.quote(subpath))) 

450 

451 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

452 self.assertEqual(child.basename(), "file?.txt") 

453 self.assertEqual(child.relativeToPathRoot, subpath) 

454 self.assertIn("%", child.path) 

455 self.assertEqual(child.unquoted_path, "/" + subpath) 

456 

457 def test_ordering(self) -> None: 

458 """Check that greater/less comparison operators work.""" 

459 a = self._make_uri("a.txt") 

460 b = self._make_uri("b/") 

461 self.assertTrue(a < b) 

462 self.assertFalse(a < a) 

463 self.assertTrue(a <= b) 

464 self.assertTrue(a <= a) 

465 self.assertTrue(b > a) 

466 self.assertFalse(b > b) 

467 self.assertTrue(b >= a) 

468 self.assertTrue(b >= b) 

469 

470 

471class GenericReadWriteTestCase(_GenericTestCase): 

472 """Test schemes that can read and write using concrete resources.""" 

473 

474 transfer_modes = ("copy", "move") 

475 testdir: Optional[str] = None 

476 

477 def setUp(self) -> None: 

478 if self.scheme is None: 

479 raise unittest.SkipTest("No scheme defined") 

480 self.root = self._make_uri("") 

481 self.root_uri = ResourcePath(self.root, forceDirectory=True, forceAbsolute=False) 

482 

483 if self.scheme == "file": 

484 # Use a local tempdir because on macOS the temp dirs use symlinks 

485 # so relsymlink gets quite confused. 

486 self.tmpdir = ResourcePath(makeTestTempDir(self.testdir)) 

487 else: 

488 # Create random tmp directory relative to the test root. 

489 self.tmpdir = self.root_uri.join( 

490 "TESTING-" + "".join(random.choices(string.ascii_lowercase + string.digits, k=8)), 

491 forceDirectory=True, 

492 ) 

493 self.tmpdir.mkdir() 

494 

495 def tearDown(self) -> None: 

496 if self.tmpdir: 

497 if self.tmpdir.isLocal: 

498 removeTestTempDir(self.tmpdir.ospath) 

499 

500 def test_file(self) -> None: 

501 uri = self.tmpdir.join("test.txt") 

502 self.assertFalse(uri.exists(), f"{uri} should not exist") 

503 self.assertTrue(uri.path.endswith("test.txt")) 

504 

505 content = "abcdefghijklmnopqrstuv\n" 

506 uri.write(content.encode()) 

507 self.assertTrue(uri.exists(), f"{uri} should now exist") 

508 self.assertEqual(uri.read().decode(), content) 

509 self.assertEqual(uri.size(), len(content.encode())) 

510 

511 with self.assertRaises(FileExistsError): 

512 uri.write(b"", overwrite=False) 

513 

514 # Not all backends can tell if a remove fails so we can not 

515 # test that a remove of a non-existent entry is guaranteed to raise. 

516 uri.remove() 

517 self.assertFalse(uri.exists()) 

518 

519 # Ideally the test would remove the file again and raise a 

520 # FileNotFoundError. This is not reliable for remote resources 

521 # and doing an explicit check before trying to remove the resource 

522 # just to raise an exception is deemed an unacceptable overhead. 

523 

524 with self.assertRaises(FileNotFoundError): 

525 uri.read() 

526 

527 with self.assertRaises(FileNotFoundError): 

528 self.tmpdir.join("file/not/there.txt").size() 

529 

530 # Check that creating a URI from a URI returns the same thing 

531 uri2 = ResourcePath(uri) 

532 self.assertEqual(uri, uri2) 

533 self.assertEqual(id(uri), id(uri2)) 

534 

535 def test_mkdir(self) -> None: 

536 newdir = self.tmpdir.join("newdir/seconddir", forceDirectory=True) 

537 newdir.mkdir() 

538 self.assertTrue(newdir.exists()) 

539 self.assertEqual(newdir.size(), 0) 

540 

541 newfile = newdir.join("temp.txt") 

542 newfile.write("Data".encode()) 

543 self.assertTrue(newfile.exists()) 

544 

545 file = self.tmpdir.join("file.txt") 

546 # Some schemes will realize that the URI is not a file and so 

547 # will raise NotADirectoryError. The file scheme is more permissive 

548 # and lets you write anything but will raise NotADirectoryError 

549 # if a non-directory is already there. We therefore write something 

550 # to the file to ensure that we trigger a portable exception. 

551 file.write(b"") 

552 with self.assertRaises(NotADirectoryError): 

553 file.mkdir() 

554 

555 # The root should exist. 

556 self.root_uri.mkdir() 

557 self.assertTrue(self.root_uri.exists()) 

558 

559 def test_transfer(self) -> None: 

560 src = self.tmpdir.join("test.txt") 

561 content = "Content is some content\nwith something to say\n\n" 

562 src.write(content.encode()) 

563 

564 can_move = "move" in self.transfer_modes 

565 for mode in self.transfer_modes: 

566 if mode == "move": 

567 continue 

568 

569 dest = self.tmpdir.join(f"dest_{mode}.txt") 

570 # Ensure that we get some debugging output. 

571 with self.assertLogs("lsst.resources", level=logging.DEBUG) as cm: 

572 dest.transfer_from(src, transfer=mode) 

573 self.assertIn("Transferring ", "\n".join(cm.output)) 

574 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

575 

576 new_content = dest.read().decode() 

577 self.assertEqual(new_content, content) 

578 

579 if mode in ("symlink", "relsymlink"): 

580 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

581 

582 # If the source and destination are hardlinks of each other 

583 # the transfer should work even if overwrite=False. 

584 if mode in ("link", "hardlink"): 

585 dest.transfer_from(src, transfer=mode) 

586 else: 

587 with self.assertRaises( 

588 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

589 ): 

590 dest.transfer_from(src, transfer=mode) 

591 

592 # Transfer again and overwrite. 

593 dest.transfer_from(src, transfer=mode, overwrite=True) 

594 

595 dest.remove() 

596 

597 b = src.read() 

598 self.assertEqual(b.decode(), new_content) 

599 

600 nbytes = 10 

601 subset = src.read(size=nbytes) 

602 self.assertEqual(len(subset), nbytes) 

603 self.assertEqual(subset.decode(), content[:nbytes]) 

604 

605 # Transferring to self should be okay. 

606 src.transfer_from(src, "auto") 

607 

608 with self.assertRaises(ValueError): 

609 src.transfer_from(src, transfer="unknown") 

610 

611 # A move transfer is special. 

612 if can_move: 

613 dest.transfer_from(src, transfer="move") 

614 self.assertFalse(src.exists()) 

615 self.assertTrue(dest.exists()) 

616 else: 

617 src.remove() 

618 

619 dest.remove() 

620 with self.assertRaises(FileNotFoundError): 

621 dest.transfer_from(src, "auto") 

622 

623 def test_local_transfer(self) -> None: 

624 """Test we can transfer to and from local file.""" 

625 remote_src = self.tmpdir.join("src.json") 

626 remote_src.write(b"42") 

627 remote_dest = self.tmpdir.join("dest.json") 

628 

629 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

630 self.assertTrue(tmp.isLocal) 

631 tmp.transfer_from(remote_src, transfer="auto") 

632 self.assertEqual(tmp.read(), remote_src.read()) 

633 

634 remote_dest.transfer_from(tmp, transfer="auto") 

635 self.assertEqual(remote_dest.read(), tmp.read()) 

636 

637 # Temporary (possibly remote) resource. 

638 # Transfers between temporary resources. 

639 with ResourcePath.temporary_uri(prefix=self.tmpdir.join("tmp"), suffix=".json") as remote_tmp: 

640 # Temporary local resource. 

641 with ResourcePath.temporary_uri(suffix=".json") as local_tmp: 

642 remote_tmp.write(b"42") 

643 if not remote_tmp.isLocal: 

644 for transfer in ("link", "symlink", "hardlink", "relsymlink"): 

645 with self.assertRaises(RuntimeError): 

646 # Trying to symlink a remote resource is not going 

647 # to work. A hardlink could work but would rely 

648 # on the local temp space being on the same 

649 # filesystem as the target. 

650 local_tmp.transfer_from(remote_tmp, transfer) 

651 local_tmp.transfer_from(remote_tmp, "move") 

652 self.assertFalse(remote_tmp.exists()) 

653 remote_tmp.transfer_from(local_tmp, "auto", overwrite=True) 

654 self.assertEqual(local_tmp.read(), remote_tmp.read()) 

655 

656 # Transfer of missing remote. 

657 remote_tmp.remove() 

658 with self.assertRaises(FileNotFoundError): 

659 local_tmp.transfer_from(remote_tmp, "auto", overwrite=True) 

660 

661 def test_local(self) -> None: 

662 """Check that remote resources can be made local.""" 

663 src = self.tmpdir.join("test.txt") 

664 original_content = "Content is some content\nwith something to say\n\n" 

665 src.write(original_content.encode()) 

666 

667 # Run this twice to ensure use of cache in code coverage 

668 # if applicable. 

669 for _ in (1, 2): 

670 with src.as_local() as local_uri: 

671 self.assertTrue(local_uri.isLocal) 

672 content = local_uri.read().decode() 

673 self.assertEqual(content, original_content) 

674 

675 if src.isLocal: 

676 self.assertEqual(src, local_uri) 

677 

678 with self.assertRaises(IsADirectoryError): 

679 with self.root_uri.as_local() as local_uri: 

680 pass 

681 

682 def test_walk(self) -> None: 

683 """Walk a directory hierarchy.""" 

684 root = self.tmpdir.join("walk/") 

685 

686 # Look for a file that is not there 

687 file = root.join("config/basic/butler.yaml") 

688 found_list = list(ResourcePath.findFileResources([file])) 

689 self.assertEqual(found_list[0], file) 

690 

691 # First create the files (content is irrelevant). 

692 expected_files = { 

693 "dir1/a.yaml", 

694 "dir1/b.yaml", 

695 "dir1/c.json", 

696 "dir2/d.json", 

697 "dir2/e.yaml", 

698 } 

699 expected_uris = {root.join(f) for f in expected_files} 

700 for uri in expected_uris: 

701 uri.write(b"") 

702 self.assertTrue(uri.exists()) 

703 

704 # Look for the files. 

705 found = set(ResourcePath.findFileResources([root])) 

706 self.assertEqual(found, expected_uris) 

707 

708 # Now solely the YAML files. 

709 expected_yaml = {u for u in expected_uris if u.getExtension() == ".yaml"} 

710 found = set(ResourcePath.findFileResources([root], file_filter=r".*\.yaml$")) 

711 self.assertEqual(found, expected_yaml) 

712 

713 # Now two explicit directories and a file 

714 expected = set(u for u in expected_yaml) 

715 expected.add(file) 

716 

717 found = set( 

718 ResourcePath.findFileResources( 

719 [file, root.join("dir1/"), root.join("dir2/")], 

720 file_filter=r".*\.yaml$", 

721 ) 

722 ) 

723 self.assertEqual(found, expected) 

724 

725 # Group by directory -- find everything and compare it with what