Coverage for tests/test_uri.py: 11%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

594 statements  

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import glob 

13import os 

14import pathlib 

15import shutil 

16import unittest 

17import urllib.parse 

18import uuid 

19 

20import responses 

21 

22try: 

23 import boto3 

24 import botocore 

25 from moto import mock_s3 

26except ImportError: 

27 boto3 = None 

28 

29 def mock_s3(cls): 

30 """A no-op decorator in case moto mock_s3 can not be imported.""" 

31 return cls 

32 

33 

34import lsst.resources 

35from lsst.resources import ResourcePath 

36from lsst.resources.s3utils import setAwsEnvCredentials, unsetAwsEnvCredentials 

37from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

38 

39TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

40 

41 

42def _check_open(test_case, uri, *, mode_suffixes=("", "t", "b"), **kwargs) -> None: 

43 """Test an implementation of ButlerURI.open. 

44 

45 Parameters 

46 ---------- 

47 test_case : `unittest.TestCase` 

48 Test case to use for assertions. 

49 uri : `ButlerURI` 

50 URI to use for tests. Must point to a writeable location that is not 

51 yet occupied by a file. On return, the location may point to a file 

52 only if the test fails. 

53 mode_suffixes : `Iterable` of `str` 

54 Suffixes to pass as part of the ``mode`` argument to `ButlerURI.open`, 

55 indicating whether to open as binary or as text; the only permitted 

56 elements are ``""``, ``"t"``, and ``""b""`. 

57 **kwargs 

58 Additional keyword arguments to forward to all calls to `open`. 

59 """ 

60 text_content = "wxyz🙂" 

61 bytes_content = uuid.uuid4().bytes 

62 content_by_mode_suffix = { 

63 "": text_content, 

64 "t": text_content, 

65 "b": bytes_content, 

66 } 

67 empty_content_by_mode_suffix = { 

68 "": "", 

69 "t": "", 

70 "b": b"", 

71 } 

72 for mode_suffix in mode_suffixes: 

73 content = content_by_mode_suffix[mode_suffix] 

74 # Create file with mode='x', which prohibits overwriting. 

75 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

76 write_buffer.write(content) 

77 test_case.assertTrue(uri.exists()) 

78 # Check that opening with 'x' now raises, and does not modify content. 

79 with test_case.assertRaises(FileExistsError): 

80 with uri.open("x" + mode_suffix, **kwargs) as write_buffer: 

81 write_buffer.write("bad") 

82 # Read the file we created and check the contents. 

83 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

84 test_case.assertEqual(read_buffer.read(), content) 

85 # Write two copies of the content, overwriting the single copy there. 

86 with uri.open("w" + mode_suffix, **kwargs) as write_buffer: 

87 write_buffer.write(content + content) 

88 # Read again, this time use mode='r+', which reads what is there and 

89 # then lets us write more; we'll use that to reset the file to one 

90 # copy of the content. 

91 with uri.open("r+" + mode_suffix, **kwargs) as rw_buffer: 

92 test_case.assertEqual(rw_buffer.read(), content + content) 

93 rw_buffer.seek(0) 

94 rw_buffer.truncate() 

95 rw_buffer.write(content) 

96 rw_buffer.seek(0) 

97 test_case.assertEqual(rw_buffer.read(), content) 

98 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

99 test_case.assertEqual(read_buffer.read(), content) 

100 # Append some more content to the file; should now have two copies. 

101 with uri.open("a" + mode_suffix, **kwargs) as append_buffer: 

102 append_buffer.write(content) 

103 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

104 test_case.assertEqual(read_buffer.read(), content + content) 

105 # Final mode to check is w+, which does read/write but truncates first. 

106 with uri.open("w+" + mode_suffix, **kwargs) as rw_buffer: 

107 test_case.assertEqual(rw_buffer.read(), empty_content_by_mode_suffix[mode_suffix]) 

108 rw_buffer.write(content) 

109 rw_buffer.seek(0) 

110 test_case.assertEqual(rw_buffer.read(), content) 

111 with uri.open("r" + mode_suffix, **kwargs) as read_buffer: 

112 test_case.assertEqual(read_buffer.read(), content) 

113 # Remove file to make room for the next loop of tests with this URI. 

114 uri.remove() 

115 

116 

117class FileURITestCase(unittest.TestCase): 

118 """Concrete tests for local files.""" 

119 

120 def setUp(self): 

121 # Use a local tempdir because on macOS the temp dirs use symlinks 

122 # so relsymlink gets quite confused. 

123 self.tmpdir = makeTestTempDir(TESTDIR) 

124 

125 def tearDown(self): 

126 removeTestTempDir(self.tmpdir) 

127 

128 def testFile(self): 

129 file = os.path.join(self.tmpdir, "test.txt") 

130 uri = ResourcePath(file) 

131 self.assertFalse(uri.exists(), f"{uri} should not exist") 

132 self.assertEqual(uri.ospath, file) 

133 

134 path = pathlib.Path(file) 

135 uri = ResourcePath(path) 

136 self.assertEqual(uri.ospath, file) 

137 

138 content = "abcdefghijklmnopqrstuv\n" 

139 uri.write(content.encode()) 

140 self.assertTrue(os.path.exists(file), "File should exist locally") 

141 self.assertTrue(uri.exists(), f"{uri} should now exist") 

142 self.assertEqual(uri.read().decode(), content) 

143 self.assertEqual(uri.size(), len(content.encode())) 

144 

145 with self.assertRaises(FileNotFoundError): 

146 ResourcePath("file/not/there.txt").size() 

147 

148 # Check that creating a URI from a URI returns the same thing 

149 uri2 = ResourcePath(uri) 

150 self.assertEqual(uri, uri2) 

151 self.assertEqual(id(uri), id(uri2)) 

152 

153 with self.assertRaises(ValueError): 

154 # Scheme-less URIs are not allowed to support non-file roots 

155 # at the present time. This may change in the future to become 

156 # equivalent to ResourcePath.join() 

157 ResourcePath("a/b.txt", root=ResourcePath("s3://bucket/a/b/")) 

158 

159 def testExtension(self): 

160 file = ResourcePath(os.path.join(self.tmpdir, "test.txt")) 

161 self.assertEqual(file.updatedExtension(None), file) 

162 self.assertEqual(file.updatedExtension(".txt"), file) 

163 self.assertEqual(id(file.updatedExtension(".txt")), id(file)) 

164 

165 fits = file.updatedExtension(".fits.gz") 

166 self.assertEqual(fits.basename(), "test.fits.gz") 

167 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

168 

169 def testRelative(self): 

170 """Check that we can get subpaths back from two URIs""" 

171 parent = ResourcePath(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

172 self.assertTrue(parent.isdir()) 

173 child = ResourcePath(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

174 

175 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

176 

177 not_child = ResourcePath("/a/b/dir1/file.txt") 

178 self.assertIsNone(not_child.relative_to(parent)) 

179 self.assertFalse(not_child.isdir()) 

180 

181 not_directory = ResourcePath(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

182 self.assertIsNone(child.relative_to(not_directory)) 

183 

184 # Relative URIs 

185 parent = ResourcePath("a/b/", forceAbsolute=False) 

186 child = ResourcePath("a/b/c/d.txt", forceAbsolute=False) 

187 self.assertFalse(child.scheme) 

188 self.assertEqual(child.relative_to(parent), "c/d.txt") 

189 

190 # forceAbsolute=True should work even on an existing ResourcePath 

191 self.assertTrue(pathlib.Path(ResourcePath(child, forceAbsolute=True).ospath).is_absolute()) 

192 

193 # File URI and schemeless URI 

194 parent = ResourcePath("file:/a/b/c/") 

195 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

196 

197 # If the child is relative and the parent is absolute we assume 

198 # that the child is a child of the parent unless it uses ".." 

199 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

200 

201 child = ResourcePath("../e/f/g.txt", forceAbsolute=False) 

202 self.assertIsNone(child.relative_to(parent)) 

203 

204 child = ResourcePath("../c/e/f/g.txt", forceAbsolute=False) 

205 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

206 

207 # Test non-file root with relative path. 

208 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

209 parent = ResourcePath("s3://hello/a/b/c/") 

210 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

211 

212 # Test with different netloc 

213 child = ResourcePath("http://my.host/a/b/c.txt") 

214 parent = ResourcePath("http://other.host/a/") 

215 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

216 

217 # Schemeless absolute child. 

218 # Schemeless absolute URI is constructed using root= parameter. 

219 parent = ResourcePath("file:///a/b/c/") 

220 child = ResourcePath("d/e.txt", root=parent) 

221 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

222 

223 parent = ResourcePath("c/", root="/a/b/") 

224 self.assertEqual(child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})") 

225 

226 # Absolute schemeless child with relative parent will always fail. 

227 parent = ResourcePath("d/e.txt", forceAbsolute=False) 

228 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

229 

230 def testParents(self): 

231 """Test of splitting and parent walking.""" 

232 parent = ResourcePath(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

233 child_file = parent.join("subdir/file.txt") 

234 self.assertFalse(child_file.isdir()) 

235 child_subdir, file = child_file.split() 

236 self.assertEqual(file, "file.txt") 

237 self.assertTrue(child_subdir.isdir()) 

238 self.assertEqual(child_file.dirname(), child_subdir) 

239 self.assertEqual(child_file.basename(), file) 

240 self.assertEqual(child_file.parent(), child_subdir) 

241 derived_parent = child_subdir.parent() 

242 self.assertEqual(derived_parent, parent) 

243 self.assertTrue(derived_parent.isdir()) 

244 self.assertEqual(child_file.parent().parent(), parent) 

245 

246 def testEnvVar(self): 

247 """Test that environment variables are expanded.""" 

248 

249 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

250 uri = ResourcePath("${MY_TEST_DIR}/d.txt") 

251 self.assertEqual(uri.path, "/a/b/c/d.txt") 

252 self.assertEqual(uri.scheme, "file") 

253 

254 # This will not expand 

255 uri = ResourcePath("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

256 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

257 self.assertFalse(uri.scheme) 

258 

259 def testMkdir(self): 

260 tmpdir = ResourcePath(self.tmpdir) 

261 newdir = tmpdir.join("newdir/seconddir") 

262 newdir.mkdir() 

263 self.assertTrue(newdir.exists()) 

264 newfile = newdir.join("temp.txt") 

265 newfile.write("Data".encode()) 

266 self.assertTrue(newfile.exists()) 

267 

268 def testTransfer(self): 

269 src = ResourcePath(os.path.join(self.tmpdir, "test.txt")) 

270 content = "Content is some content\nwith something to say\n\n" 

271 src.write(content.encode()) 

272 

273 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

274 dest = ResourcePath(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

275 dest.transfer_from(src, transfer=mode) 

276 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

277 

278 with open(dest.ospath, "r") as fh: 

279 new_content = fh.read() 

280 self.assertEqual(new_content, content) 

281 

282 if mode in ("symlink", "relsymlink"): 

283 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

284 

285 # If the source and destination are hardlinks of each other 

286 # the transfer should work even if overwrite=False. 

287 if mode in ("link", "hardlink"): 

288 dest.transfer_from(src, transfer=mode) 

289 else: 

290 with self.assertRaises( 

291 FileExistsError, msg=f"Overwrite of {dest} should not be allowed ({mode})" 

292 ): 

293 dest.transfer_from(src, transfer=mode) 

294 

295 dest.transfer_from(src, transfer=mode, overwrite=True) 

296 

297 os.remove(dest.ospath) 

298 

299 b = src.read() 

300 self.assertEqual(b.decode(), new_content) 

301 

302 nbytes = 10 

303 subset = src.read(size=nbytes) 

304 self.assertEqual(len(subset), nbytes) 

305 self.assertEqual(subset.decode(), content[:nbytes]) 

306 

307 with self.assertRaises(ValueError): 

308 src.transfer_from(src, transfer="unknown") 

309 

310 def testTransferIdentical(self): 

311 """Test overwrite of identical files.""" 

312 dir1 = ResourcePath(os.path.join(self.tmpdir, "dir1"), forceDirectory=True) 

313 dir1.mkdir() 

314 dir2 = os.path.join(self.tmpdir, "dir2") 

315 os.symlink(dir1.ospath, dir2) 

316 

317 # Write a test file. 

318 src_file = dir1.join("test.txt") 

319 content = "0123456" 

320 src_file.write(content.encode()) 

321 

322 # Construct URI to destination that should be identical. 

323 dest_file = ResourcePath(os.path.join(dir2), forceDirectory=True).join("test.txt") 

324 self.assertTrue(dest_file.exists()) 

325 self.assertNotEqual(src_file, dest_file) 

326 

327 # Transfer it over itself. 

328 dest_file.transfer_from(src_file, transfer="symlink", overwrite=True) 

329 new_content = dest_file.read().decode() 

330 self.assertEqual(content, new_content) 

331 

332 def testResource(self): 

333 # No resources in this package so need a resource in the main 

334 # python distribution. 

335 u = ResourcePath("resource://idlelib/Icons/README.txt") 

336 self.assertTrue(u.exists(), f"Check {u} exists") 

337 

338 content = u.read().decode() 

339 self.assertIn("IDLE", content) 

340 

341 truncated = u.read(size=9).decode() 

342 self.assertEqual(truncated, content[:9]) 

343 

344 d = ResourcePath("resource://idlelib/Icons", forceDirectory=True) 

345 self.assertTrue(u.exists(), f"Check directory {d} exists") 

346 

347 j = d.join("README.txt") 

348 self.assertEqual(u, j) 

349 self.assertFalse(j.dirLike) 

350 self.assertFalse(j.isdir()) 

351 not_there = d.join("not-there.yaml") 

352 self.assertFalse(not_there.exists()) 

353 

354 bad = ResourcePath("resource://bad.module/not.yaml") 

355 multi = ResourcePath.mexists([u, bad, not_there]) 

356 self.assertTrue(multi[u]) 

357 self.assertFalse(multi[bad]) 

358 self.assertFalse(multi[not_there]) 

359 

360 def testEscapes(self): 

361 """Special characters in file paths""" 

362 src = ResourcePath("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

363 self.assertFalse(src.scheme) 

364 src.write(b"Some content") 

365 self.assertTrue(src.exists()) 

366 

367 # abspath always returns a file scheme 

368 file = src.abspath() 

369 self.assertTrue(file.exists()) 

370 self.assertIn("???", file.ospath) 

371 self.assertNotIn("???", file.path) 

372 

373 file = file.updatedFile("tests??.txt") 

374 self.assertNotIn("??.txt", file.path) 

375 file.write(b"Other content") 

376 self.assertEqual(file.read(), b"Other content") 

377 

378 src = src.updatedFile("tests??.txt") 

379 self.assertIn("??.txt", src.path) 

380 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

381 

382 # File URI and schemeless URI 

383 parent = ResourcePath("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

384 child = ResourcePath("e/f/g.txt", forceAbsolute=False) 

385 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

386 

387 child = ResourcePath("e/f??#/g.txt", forceAbsolute=False) 

388 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

389 

390 child = ResourcePath("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

391 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

392 

393 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

394 

395 # Schemeless so should not quote 

396 dir = ResourcePath("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

397 self.assertIn("???", dir.ospath) 

398 self.assertIn("???", dir.path) 

399 self.assertFalse(dir.scheme) 

400 

401 # dir.join() morphs into a file scheme 

402 new = dir.join("test_j.txt") 

403 self.assertIn("???", new.ospath, f"Checking {new}") 

404 new.write(b"Content") 

405 

406 new2name = "###/test??.txt" 

407 new2 = dir.join(new2name) 

408 self.assertIn("???", new2.ospath) 

409 new2.write(b"Content") 

410 self.assertTrue(new2.ospath.endswith(new2name)) 

411 self.assertEqual(new.read(), new2.read()) 

412 

413 fdir = dir.abspath() 

414 self.assertNotIn("???", fdir.path) 

415 self.assertIn("???", fdir.ospath) 

416 self.assertEqual(fdir.scheme, "file") 

417 fnew = dir.join("test_jf.txt") 

418 fnew.write(b"Content") 

419 

420 fnew2 = fdir.join(new2name) 

421 fnew2.write(b"Content") 

422 self.assertTrue(fnew2.ospath.endswith(new2name)) 

423 self.assertNotIn("###", fnew2.path) 

424 

425 self.assertEqual(fnew.read(), fnew2.read()) 

426 

427 # Test that children relative to schemeless and file schemes 

428 # still return the same unquoted name 

429 self.assertEqual(fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})") 

430 self.assertEqual(fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})") 

431 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})") 

432 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

433 

434 # Check for double quoting 

435 plus_path = "/a/b/c+d/" 

436 with self.assertLogs(level="WARNING"): 

437 uri = ResourcePath(urllib.parse.quote(plus_path), forceDirectory=True) 

438 self.assertEqual(uri.ospath, plus_path) 

439 

440 # Check that # is not escaped for schemeless URIs 

441 hash_path = "/a/b#/c&d#xyz" 

442 hpos = hash_path.rfind("#") 

443 uri = ResourcePath(hash_path) 

444 self.assertEqual(uri.ospath, hash_path[:hpos]) 

445 self.assertEqual(uri.fragment, hash_path[hpos + 1 :]) 

446 

447 def testHash(self): 

448 """Test that we can store URIs in sets and as keys.""" 

449 uri1 = ResourcePath(TESTDIR) 

450 uri2 = uri1.join("test/") 

451 s = {uri1, uri2} 

452 self.assertIn(uri1, s) 

453 

454 d = {uri1: "1", uri2: "2"} 

455 self.assertEqual(d[uri2], "2") 

456 

457 def testWalk(self): 

458 """Test ResourcePath.walk().""" 

459 test_dir_uri = ResourcePath(TESTDIR) 

460 

461 # Look for a file that is not there 

462 file = test_dir_uri.join("config/basic/butler.yaml") 

463 found = list(ResourcePath.findFileResources([file])) 

464 self.assertEqual(found[0], file) 

465 

466 # Compare against the full local paths 

467 expected = set( 

468 p for p in glob.glob(os.path.join(TESTDIR, "data", "**"), recursive=True) if os.path.isfile(p) 

469 ) 

470 found = set(u.ospath for u in ResourcePath.findFileResources([test_dir_uri.join("data")])) 

471 self.assertEqual(found, expected) 

472 

473 # Now solely the YAML files 

474 expected_yaml = set(glob.glob(os.path.join(TESTDIR, "data", "**", "*.yaml"), recursive=True)) 

475 found = set( 

476 u.ospath 

477 for u in ResourcePath.findFileResources([test_dir_uri.join("data")], file_filter=r".*\.yaml$") 

478 ) 

479 self.assertEqual(found, expected_yaml) 

480 

481 # Now two explicit directories and a file 

482 expected = set(glob.glob(os.path.join(TESTDIR, "data", "dir1", "*.yaml"), recursive=True)) 

483 expected.update(set(glob.glob(os.path.join(TESTDIR, "data", "dir2", "*.yaml"), recursive=True))) 

484 expected.add(file.ospath) 

485 

486 found = set( 

487 u.ospath 

488 for u in ResourcePath.findFileResources( 

489 [file, test_dir_uri.join("data/dir1"), test_dir_uri.join("data/dir2")], 

490 file_filter=r".*\.yaml$", 

491 ) 

492 ) 

493 self.assertEqual(found, expected) 

494 

495 # Group by directory -- find everything and compare it with what 

496 # we expected to be there in total. 

497 found_yaml = set() 

498 counter = 0 

499 for uris in ResourcePath.findFileResources( 

500 [file, test_dir_uri.join("data/")], file_filter=r".*\.yaml$", grouped=True 

501 ): 

502 found = set(u.ospath for u in uris) 

503 if found: 

504 counter += 1 

505 

506 found_yaml.update(found) 

507 

508 expected_yaml_2 = expected_yaml 

509 expected_yaml_2.add(file.ospath) 

510 self.assertEqual(found_yaml, expected_yaml) 

511 self.assertEqual(counter, 3) 

512 

513 # Grouping but check that single files are returned in a single group 

514 # at the end 

515 file2 = test_dir_uri.join("config/templates/templates-bad.yaml") 

516 found = list( 

517 ResourcePath.findFileResources([file, file2, test_dir_uri.join("data/dir2")], grouped=True) 

518 ) 

519 self.assertEqual(len(found), 2) 

520 self.assertEqual(list(found[1]), [file, file2]) 

521 

522 with self.assertRaises(ValueError): 

523 list(file.walk()) 

524 

525 def testRootURI(self): 

526 """Test ResourcePath.root_uri().""" 

527 uri = ResourcePath("https://www.notexist.com:8080/file/test") 

528 uri2 = ResourcePath("s3://www.notexist.com/file/test") 

529 self.assertEqual(uri.root_uri().geturl(), "https://www.notexist.com:8080/") 

530 self.assertEqual(uri2.root_uri().geturl(), "s3://www.notexist.com/") 

531 

532 def testJoin(self): 

533 """Test .join method.""" 

534 

535 root_str = "s3://bucket/hsc/payload/" 

536 root = ResourcePath(root_str) 

537 

538 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

539 add_dir = root.join("b/c/d/") 

540 self.assertTrue(add_dir.isdir()) 

541 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

542 

543 up_relative = root.join("../b/c.txt") 

544 self.assertFalse(up_relative.isdir()) 

545 self.assertEqual(up_relative.geturl(), "s3://bucket/hsc/b/c.txt") 

546 

547 quote_example = "b&c.t@x#t" 

548 needs_quote = root.join(quote_example) 

549 self.assertEqual(needs_quote.unquoted_path, f"/hsc/payload/{quote_example}") 

550 

551 other = ResourcePath("file://localhost/test.txt") 

552 self.assertEqual(root.join(other), other) 

553 self.assertEqual(other.join("b/new.txt").geturl(), "file://localhost/b/new.txt") 

554 

555 joined = ResourcePath("s3://bucket/hsc/payload/").join( 

556 ResourcePath("test.qgraph", forceAbsolute=False) 

557 ) 

558 self.assertEqual(joined, ResourcePath("s3://bucket/hsc/payload/test.qgraph")) 

559 

560 with self.assertRaises(ValueError): 

561 ResourcePath("s3://bucket/hsc/payload/").join(ResourcePath("test.qgraph")) 

562 

563 def testTemporary(self): 

564 with ResourcePath.temporary_uri(suffix=".json") as tmp: 

565 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

566 self.assertTrue(tmp.isabs(), f"uri: {tmp}") 

567 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

568 tmp.write(b"abcd") 

569 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

570 self.assertTrue(tmp.isTemporary) 

571 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

572 

573 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

574 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".yaml") as tmp: 

575 # Use a specified tmpdir and check it is okay for the file 

576 # to not be created. 

577 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

578 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists") 

579 

580 def test_open(self): 

581 tmpdir = ResourcePath(self.tmpdir, forceDirectory=True) 

582 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".txt") as tmp: 

583 _check_open(self, tmp, mode_suffixes=("", "t")) 

584 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16") 

585 _check_open(self, tmp, mode_suffixes=("t",), prefer_file_temporary=True) 

586 _check_open(self, tmp, mode_suffixes=("t",), encoding="utf-16", prefer_file_temporary=True) 

587 with ResourcePath.temporary_uri(prefix=tmpdir, suffix=".dat") as tmp: 

588 _check_open(self, tmp, mode_suffixes=("b",)) 

589 _check_open(self, tmp, mode_suffixes=("b"), prefer_file_temporary=True) 

590 

591 

592@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

593@mock_s3 

594class S3URITestCase(unittest.TestCase): 

595 """Tests involving S3""" 

596 

597 bucketName = "any_bucket" 

598 """Bucket name to use in tests""" 

599 

600 def setUp(self): 

601 # Local test directory 

602 self.tmpdir = makeTestTempDir(TESTDIR) 

603 

604 # set up some fake credentials if they do not exist 

605 self.usingDummyCredentials = setAwsEnvCredentials() 

606 

607 # MOTO needs to know that we expect Bucket bucketname to exist 

608 s3 = boto3.resource("s3") 

609 s3.create_bucket(Bucket=self.bucketName) 

610 

611 def tearDown(self): 

612 s3 = boto3.resource("s3") 

613 bucket = s3.Bucket(self.bucketName) 

614 try: 

615 bucket.objects.all().delete() 

616 except botocore.exceptions.ClientError as e: 

617 if e.response["Error"]["Code"] == "404": 

618 # the key was not reachable - pass 

619 pass 

620 else: 

621 raise 

622 

623 bucket = s3.Bucket(self.bucketName) 

624 bucket.delete() 

625 

626 # unset any potentially set dummy credentials 

627 if self.usingDummyCredentials: 

628 unsetAwsEnvCredentials() 

629 

630 shutil.rmtree(self.tmpdir, ignore_errors=True) 

631 

632 def makeS3Uri(self, path): 

633 return f"s3://{self.bucketName}/{path}" 

634 

635 def testTransfer(self): 

636 src = ResourcePath(os.path.join(self.tmpdir, "test.txt")) 

637 content = "Content is some content\nwith something to say\n\n" 

638 src.write(content.encode()) 

639 self.assertTrue(src.exists()) 

640 self.assertEqual(src.size(), len(content.encode())) 

641 

642 dest = ResourcePath(self.makeS3Uri("test.txt")) 

643 self.assertFalse(dest.exists()) 

644 

645 with self.assertRaises(FileNotFoundError): 

646 dest.size() 

647 

648 dest.transfer_from(src, transfer="copy") 

649 self.assertTrue(dest.exists()) 

650 

651 dest2 = ResourcePath(self.makeS3Uri("copied.txt")) 

652 dest2.transfer_from(dest, transfer="copy") 

653 self.assertTrue(dest2.exists()) 

654 

655 local = ResourcePath(os.path.join(self.tmpdir, "copied.txt")) 

656 local.transfer_from(dest2, transfer="copy") 

657 with open(local.ospath, "r") as fd: 

658 new_content = fd.read() 

659 self.assertEqual(new_content, content) 

660 

661 with self.assertRaises(ValueError): 

662 dest2.transfer_from(local, transfer="symlink") 

663 

664 b = dest.read() 

665 self.assertEqual(b.decode(), new_content) 

666 

667 nbytes = 10 

668 subset = dest.read(size=nbytes) 

669 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

670 self.assertEqual(subset.decode(), content[:nbytes]) 

671 

672 with self.assertRaises(FileExistsError): 

673 dest.transfer_from(src, transfer="copy") 

674 

675 dest.transfer_from(src, transfer="copy", overwrite=True) 

676 

677 def testWalk(self): 

678 """Test that we can list an S3 bucket""" 

679 # Files we want to create 

680 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json") 

681 expected_uris = [ResourcePath(self.makeS3Uri(path)) for path in expected] 

682 for uri in expected_uris: 

683 # Doesn't matter what we write 

684 uri.write("123".encode()) 

685 

686 # Find all the files in the a/ tree 

687 found = set(uri.path for uri in ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("a/"))])) 

688 self.assertEqual(found, {uri.path for uri in expected_uris}) 

689 

690 # Find all the files in the a/ tree but group by folder 

691 found = ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("a/"))], grouped=True) 

692 expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt",), ("/a/b/c/d/v.json",)) 

693 

694 for got, expect in zip(found, expected): 

695 self.assertEqual(tuple(u.path for u in got), expect) 

696 

697 # Find only JSON files 

698 found = set( 

699 uri.path 

700 for uri in ResourcePath.findFileResources( 

701 [ResourcePath(self.makeS3Uri("a/"))], file_filter=r"\.json$" 

702 ) 

703 ) 

704 self.assertEqual(found, {uri.path for uri in expected_uris if uri.path.endswith(".json")}) 

705 

706 # JSON files grouped by directory 

707 found = ResourcePath.findFileResources( 

708 [ResourcePath(self.makeS3Uri("a/"))], file_filter=r"\.json$", grouped=True 

709 ) 

710 expected = (("/a/z.json",), ("/a/b/c/d/v.json",)) 

711 

712 for got, expect in zip(found, expected): 

713 self.assertEqual(tuple(u.path for u in got), expect) 

714 

715 # Check pagination works with large numbers of files. S3 API limits 

716 # us to 1000 response per list_objects call so create lots of files 

717 created = set() 

718 counter = 1 

719 n_dir1 = 1100 

720 while counter <= n_dir1: 

721 new = ResourcePath(self.makeS3Uri(f"test/file{counter:04d}.txt")) 

722 new.write(f"{counter}".encode()) 

723 created.add(str(new)) 

724 counter += 1 

725 counter = 1 

726 # Put some in a subdirectory to make sure we are looking in a 

727 # hierarchy. 

728 n_dir2 = 100 

729 while counter <= n_dir2: 

730 new = ResourcePath(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt")) 

731 new.write(f"{counter}".encode()) 

732 created.add(str(new)) 

733 counter += 1 

734 

735 found = ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("test/"))]) 

736 self.assertEqual({str(u) for u in found}, created) 

737 

738 # Again with grouping. 

739 found = list(ResourcePath.findFileResources([ResourcePath(self.makeS3Uri("test/"))], grouped=True)) 

740 self.assertEqual(len(found), 2) 

741 dir_1 = list(found[0]) 

742 dir_2 = list(found[1]) 

743 self.assertEqual(len(dir_1), n_dir1) 

744 self.assertEqual(len(dir_2), n_dir2) 

745 

746 def testWrite(self): 

747 s3write = ResourcePath(self.makeS3Uri("created.txt")) 

748 content = "abcdefghijklmnopqrstuv\n" 

749 s3write.write(content.encode()) 

750 self.assertEqual(s3write.read().decode(), content) 

751 

752 def testTemporary(self): 

753 s3root = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True) 

754 with ResourcePath.temporary_uri(prefix=s3root, suffix=".json") as tmp: 

755 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

756 self.assertEqual(tmp.scheme, "s3", f"uri: {tmp}") 

757 self.assertEqual(tmp.parent(), s3root) 

758 basename = tmp.basename() 

759 content = "abcd" 

760 tmp.write(content.encode()) 

761 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

762 self.assertFalse(tmp.exists()) 

763 

764 # Again without writing anything, to check that there is no complaint 

765 # on exit of context manager. 

766 with ResourcePath.temporary_uri(prefix=s3root, suffix=".json") as tmp: 

767 self.assertFalse(tmp.exists()) 

768 # Check that the file has a different name than before. 

769 self.assertNotEqual(tmp.basename(), basename, f"uri: {tmp}") 

770 self.assertFalse(tmp.exists()) 

771 

772 def testRelative(self): 

773 """Check that we can get subpaths back from two URIs""" 

774 parent = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True) 

775 child = ResourcePath(self.makeS3Uri("rootdir/dir1/file.txt")) 

776 

777 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

778 

779 not_child = ResourcePath(self.makeS3Uri("/a/b/dir1/file.txt")) 

780 self.assertFalse(not_child.relative_to(parent)) 

781 

782 not_s3 = ResourcePath(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

783 self.assertFalse(child.relative_to(not_s3)) 

784 

785 def testQuoting(self): 

786 """Check that quoting works.""" 

787 parent = ResourcePath(self.makeS3Uri("rootdir"), forceDirectory=True) 

788 subpath = "rootdir/dir1+/file?.txt" 

789 child = ResourcePath(self.makeS3Uri(urllib.parse.quote(subpath))) 

790 

791 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

792 self.assertEqual(child.basename(), "file?.txt") 

793 self.assertEqual(child.relativeToPathRoot, subpath) 

794 self.assertIn("%", child.path) 

795 self.assertEqual(child.unquoted_path, "/" + subpath) 

796 

797 def test_open(self): 

798 text_uri = ResourcePath(self.makeS3Uri("file.txt")) 

799 _check_open(self, text_uri, mode_suffixes=("", "t")) 

800 _check_open(self, text_uri, mode_suffixes=("t",), encoding="utf-16") 

801 _check_open(self, text_uri, mode_suffixes=("t",), prefer_file_temporary=True) 

802 _check_open(self, text_uri, mode_suffixes=("t",), prefer_file_temporary=True, encoding="utf-16") 

803 binary_uri = ResourcePath(self.makeS3Uri("file.dat")) 

804 _check_open(self, binary_uri, mode_suffixes=("b",)) 

805 _check_open(self, binary_uri, mode_suffixes=("b",), prefer_file_temporary=True) 

806 

807 

808# Mock required environment variables during tests 

809@unittest.mock.patch.dict( 

810 os.environ, 

811 { 

812 "LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

813 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join(TESTDIR, "data/webdav/token"), 

814 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs", 

815 }, 

816) 

817class WebdavURITestCase(unittest.TestCase): 

818 def setUp(self): 

819 # Local test directory 

820 self.tmpdir = makeTestTempDir(TESTDIR) 

821 

822 serverRoot = "www.not-exists.orgx" 

823 existingFolderName = "existingFolder" 

824 existingFileName = "existingFile" 

825 notExistingFileName = "notExistingFile" 

826 

827 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True) 

828 self.existingFileResourcePath = ResourcePath( 

829 f"https://{serverRoot}/{existingFolderName}/{existingFileName}" 

830 ) 

831 self.notExistingFileResourcePath = ResourcePath( 

832 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}" 

833 ) 

834 self.existingFolderResourcePath = ResourcePath( 

835 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True 

836 ) 

837 self.notExistingFolderResourcePath = ResourcePath( 

838 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True 

839 ) 

840 

841 # Need to declare the options 

842 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"}) 

843 

844 # Used by HttpResourcePath.exists() 

845 responses.add( 

846 responses.HEAD, 

847 self.existingFileResourcePath.geturl(), 

848 status=200, 

849 headers={"Content-Length": "1024"}, 

850 ) 

851 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404) 

852 

853 # Used by HttpResourcePath.read() 

854 responses.add( 

855 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!") 

856 ) 

857 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404) 

858 

859 # Used by HttpResourcePath.write() 

860 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201) 

861 

862 # Used by HttpResourcePath.transfer_from() 

863 responses.add( 

864 responses.Response( 

865 url=self.existingFileResourcePath.geturl(), 

866 method="COPY", 

867 headers={"Destination": self.existingFileResourcePath.geturl()}, 

868 status=201, 

869 ) 

870 ) 

871 responses.add( 

872 responses.Response( 

873 url=self.existingFileResourcePath.geturl(), 

874 method="COPY", 

875 headers={"Destination": self.notExistingFileResourcePath.geturl()}, 

876 status=201, 

877 ) 

878 ) 

879 responses.add( 

880 responses.Response( 

881 url=self.existingFileResourcePath.geturl(), 

882 method="MOVE", 

883 headers={"Destination": self.notExistingFileResourcePath.geturl()}, 

884 status=201, 

885 ) 

886 ) 

887 

888 # Used by HttpResourcePath.remove() 

889 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200) 

890 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404) 

891 

892 # Used by HttpResourcePath.mkdir() 

893 responses.add( 

894 responses.HEAD, 

895 self.existingFolderResourcePath.geturl(), 

896 status=200, 

897 headers={"Content-Length": "1024"}, 

898 ) 

899 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"}) 

900 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404) 

901 responses.add( 

902 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201) 

903 ) 

904 responses.add( 

905 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403) 

906 ) 

907 

908 @responses.activate 

909 def testExists(self): 

910 

911 self.assertTrue(self.existingFileResourcePath.exists()) 

912 self.assertFalse(self.notExistingFileResourcePath.exists()) 

913 

914 self.assertEqual(self.existingFileResourcePath.size(), 1024) 

915 with self.assertRaises(FileNotFoundError): 

916 self.notExistingFileResourcePath.size() 

917 

918 @responses.activate 

919 def testRemove(self): 

920 

921 self.assertIsNone(self.existingFileResourcePath.remove()) 

922 with self.assertRaises(FileNotFoundError): 

923 self.notExistingFileResourcePath.remove() 

924 

925 @responses.activate 

926 def testMkdir(self): 

927 

928 # The mock means that we can't check this now exists 

929 self.notExistingFolderResourcePath.mkdir() 

930 

931 # This should do nothing 

932 self.existingFolderResourcePath.mkdir() 

933 

934 with self.assertRaises(ValueError): 

935 self.notExistingFileResourcePath.mkdir() 

936 

937 @responses.activate 

938 def testRead(self): 

939 

940 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!") 

941 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.") 

942 with self.assertRaises(FileNotFoundError): 

943 self.notExistingFileResourcePath.read() 

944 

945 # Run this twice to ensure use of cache in code coverag. 

946 for _ in (1, 2): 

947 with self.existingFileResourcePath.as_local() as local_uri: 

948 self.assertTrue(local_uri.isLocal) 

949 content = local_uri.read().decode() 

950 self.assertEqual(content, "It works!") 

951 

952 # Check that the environment variable is being read. 

953 lsst.resources.http._TMPDIR = None 

954 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir}): 

955 with self.existingFileResourcePath.as_local() as local_uri: 

956 self.assertTrue(local_uri.isLocal) 

957 content = local_uri.read().decode() 

958 self.assertEqual(content, "It works!") 

959 self.assertIsNotNone(local_uri.relative_to(ResourcePath(self.tmpdir))) 

960 

961 @responses.activate 

962 def testWrite(self): 

963 

964 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content."))) 

965 with self.assertRaises(FileExistsError): 

966 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False) 

967 

968 @responses.activate 

969 def testTransfer(self): 

970 

971 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)) 

972 self.assertIsNone( 

973 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move") 

974 ) 

975 with self.assertRaises(FileExistsError): 

976 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath) 

977 with self.assertRaises(ValueError): 

978 self.notExistingFileResourcePath.transfer_from( 

979 src=self.existingFileResourcePath, transfer="unsupported" 

980 ) 

981 

982 def testParent(self): 

983 

984 self.assertEqual( 

985 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl() 

986 ) 

987 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl()) 

988 self.assertEqual( 

989 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl() 

990 ) 

991 

992 

993if __name__ == "__main__": 993 ↛ 994line 993 didn't jump to line 994, because the condition on line 993 was never true

994 unittest.main()