Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import glob 

23import os 

24import shutil 

25import unittest 

26import urllib.parse 

27import responses 

28 

29try: 

30 import boto3 

31 import botocore 

32 from moto import mock_s3 

33except ImportError: 

34 boto3 = None 

35 

36 def mock_s3(cls): 

37 """A no-op decorator in case moto mock_s3 can not be imported. 

38 """ 

39 return cls 

40 

41from lsst.daf.butler import ButlerURI 

42from lsst.daf.butler.core._butlerUri.s3utils import (setAwsEnvCredentials, 

43 unsetAwsEnvCredentials) 

44from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

45 

46TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49class FileURITestCase(unittest.TestCase): 

50 """Concrete tests for local files""" 

51 

52 def setUp(self): 

53 # Use a local tempdir because on macOS the temp dirs use symlinks 

54 # so relsymlink gets quite confused. 

55 self.tmpdir = makeTestTempDir(TESTDIR) 

56 

57 def tearDown(self): 

58 removeTestTempDir(self.tmpdir) 

59 

60 def testFile(self): 

61 file = os.path.join(self.tmpdir, "test.txt") 

62 uri = ButlerURI(file) 

63 self.assertFalse(uri.exists(), f"{uri} should not exist") 

64 self.assertEqual(uri.ospath, file) 

65 

66 content = "abcdefghijklmnopqrstuv\n" 

67 uri.write(content.encode()) 

68 self.assertTrue(os.path.exists(file), "File should exist locally") 

69 self.assertTrue(uri.exists(), f"{uri} should now exist") 

70 self.assertEqual(uri.read().decode(), content) 

71 self.assertEqual(uri.size(), len(content.encode())) 

72 

73 with self.assertRaises(FileNotFoundError): 

74 ButlerURI("file/not/there.txt").size() 

75 

76 # Check that creating a URI from a URI returns the same thing 

77 uri2 = ButlerURI(uri) 

78 self.assertEqual(uri, uri2) 

79 self.assertEqual(id(uri), id(uri2)) 

80 

81 def testExtension(self): 

82 file = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

83 self.assertEqual(file.updatedExtension(None), file) 

84 self.assertEqual(file.updatedExtension(".txt"), file) 

85 self.assertEqual(id(file.updatedExtension(".txt")), id(file)) 

86 

87 fits = file.updatedExtension(".fits.gz") 

88 self.assertEqual(fits.basename(), "test.fits.gz") 

89 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

90 

91 def testRelative(self): 

92 """Check that we can get subpaths back from two URIs""" 

93 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

94 self.assertTrue(parent.isdir()) 

95 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

96 

97 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

98 

99 not_child = ButlerURI("/a/b/dir1/file.txt") 

100 self.assertFalse(not_child.relative_to(parent)) 

101 self.assertFalse(not_child.isdir()) 

102 

103 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

104 self.assertFalse(child.relative_to(not_directory)) 

105 

106 # Relative URIs 

107 parent = ButlerURI("a/b/", forceAbsolute=False) 

108 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

109 self.assertFalse(child.scheme) 

110 self.assertEqual(child.relative_to(parent), "c/d.txt") 

111 

112 # File URI and schemeless URI 

113 parent = ButlerURI("file:/a/b/c/") 

114 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

115 

116 # If the child is relative and the parent is absolute we assume 

117 # that the child is a child of the parent unless it uses ".." 

118 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

119 

120 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

121 self.assertFalse(child.relative_to(parent)) 

122 

123 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

124 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

125 

126 def testParents(self): 

127 """Test of splitting and parent walking.""" 

128 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

129 child_file = parent.join("subdir/file.txt") 

130 self.assertFalse(child_file.isdir()) 

131 child_subdir, file = child_file.split() 

132 self.assertEqual(file, "file.txt") 

133 self.assertTrue(child_subdir.isdir()) 

134 self.assertEqual(child_file.dirname(), child_subdir) 

135 self.assertEqual(child_file.basename(), file) 

136 self.assertEqual(child_file.parent(), child_subdir) 

137 derived_parent = child_subdir.parent() 

138 self.assertEqual(derived_parent, parent) 

139 self.assertTrue(derived_parent.isdir()) 

140 self.assertEqual(child_file.parent().parent(), parent) 

141 

142 def testEnvVar(self): 

143 """Test that environment variables are expanded.""" 

144 

145 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

146 uri = ButlerURI("${MY_TEST_DIR}/d.txt") 

147 self.assertEqual(uri.path, "/a/b/c/d.txt") 

148 self.assertEqual(uri.scheme, "file") 

149 

150 # This will not expand 

151 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

152 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

153 self.assertFalse(uri.scheme) 

154 

155 def testMkdir(self): 

156 tmpdir = ButlerURI(self.tmpdir) 

157 newdir = tmpdir.join("newdir/seconddir") 

158 newdir.mkdir() 

159 self.assertTrue(newdir.exists()) 

160 newfile = newdir.join("temp.txt") 

161 newfile.write("Data".encode()) 

162 self.assertTrue(newfile.exists()) 

163 

164 def testTransfer(self): 

165 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

166 content = "Content is some content\nwith something to say\n\n" 

167 src.write(content.encode()) 

168 

169 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

170 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

171 dest.transfer_from(src, transfer=mode) 

172 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

173 

174 with open(dest.ospath, "r") as fh: 

175 new_content = fh.read() 

176 self.assertEqual(new_content, content) 

177 

178 if mode in ("symlink", "relsymlink"): 

179 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

180 

181 with self.assertRaises(FileExistsError): 

182 dest.transfer_from(src, transfer=mode) 

183 

184 dest.transfer_from(src, transfer=mode, overwrite=True) 

185 

186 os.remove(dest.ospath) 

187 

188 b = src.read() 

189 self.assertEqual(b.decode(), new_content) 

190 

191 nbytes = 10 

192 subset = src.read(size=nbytes) 

193 self.assertEqual(len(subset), nbytes) 

194 self.assertEqual(subset.decode(), content[:nbytes]) 

195 

196 with self.assertRaises(ValueError): 

197 src.transfer_from(src, transfer="unknown") 

198 

199 def testResource(self): 

200 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

201 self.assertTrue(u.exists(), f"Check {u} exists") 

202 

203 content = u.read().decode() 

204 self.assertTrue(content.startswith("datastore:")) 

205 

206 truncated = u.read(size=9).decode() 

207 self.assertEqual(truncated, "datastore") 

208 

209 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

210 self.assertTrue(u.exists(), f"Check directory {d} exists") 

211 

212 j = d.join("datastore.yaml") 

213 self.assertEqual(u, j) 

214 self.assertFalse(j.dirLike) 

215 self.assertFalse(j.isdir()) 

216 self.assertFalse(d.join("not-there.yaml").exists()) 

217 

218 def testEscapes(self): 

219 """Special characters in file paths""" 

220 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

221 self.assertFalse(src.scheme) 

222 src.write(b"Some content") 

223 self.assertTrue(src.exists()) 

224 

225 # abspath always returns a file scheme 

226 file = src.abspath() 

227 self.assertTrue(file.exists()) 

228 self.assertIn("???", file.ospath) 

229 self.assertNotIn("???", file.path) 

230 

231 file = file.updatedFile("tests??.txt") 

232 self.assertNotIn("??.txt", file.path) 

233 file.write(b"Other content") 

234 self.assertEqual(file.read(), b"Other content") 

235 

236 src = src.updatedFile("tests??.txt") 

237 self.assertIn("??.txt", src.path) 

238 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

239 

240 # File URI and schemeless URI 

241 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

242 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

243 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

244 

245 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

246 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

247 

248 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

249 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

250 

251 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

252 

253 # Schemeless so should not quote 

254 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

255 self.assertIn("???", dir.ospath) 

256 self.assertIn("???", dir.path) 

257 self.assertFalse(dir.scheme) 

258 

259 # dir.join() morphs into a file scheme 

260 new = dir.join("test_j.txt") 

261 self.assertIn("???", new.ospath, f"Checking {new}") 

262 new.write(b"Content") 

263 

264 new2name = "###/test??.txt" 

265 new2 = dir.join(new2name) 

266 self.assertIn("???", new2.ospath) 

267 new2.write(b"Content") 

268 self.assertTrue(new2.ospath.endswith(new2name)) 

269 self.assertEqual(new.read(), new2.read()) 

270 

271 fdir = dir.abspath() 

272 self.assertNotIn("???", fdir.path) 

273 self.assertIn("???", fdir.ospath) 

274 self.assertEqual(fdir.scheme, "file") 

275 fnew = dir.join("test_jf.txt") 

276 fnew.write(b"Content") 

277 

278 fnew2 = fdir.join(new2name) 

279 fnew2.write(b"Content") 

280 self.assertTrue(fnew2.ospath.endswith(new2name)) 

281 self.assertNotIn("###", fnew2.path) 

282 

283 self.assertEqual(fnew.read(), fnew2.read()) 

284 

285 # Test that children relative to schemeless and file schemes 

286 # still return the same unquoted name 

287 self.assertEqual(fnew2.relative_to(fdir), new2name) 

288 self.assertEqual(fnew2.relative_to(dir), new2name) 

289 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

290 self.assertEqual(new2.relative_to(dir), new2name) 

291 

292 # Check for double quoting 

293 plus_path = "/a/b/c+d/" 

294 with self.assertLogs(level="WARNING"): 

295 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

296 self.assertEqual(uri.ospath, plus_path) 

297 

298 # Check that # is not escaped for schemeless URIs 

299 hash_path = "/a/b#/c&d#xyz" 

300 hpos = hash_path.rfind("#") 

301 uri = ButlerURI(hash_path) 

302 self.assertEqual(uri.ospath, hash_path[:hpos]) 

303 self.assertEqual(uri.fragment, hash_path[hpos + 1:]) 

304 

305 def testHash(self): 

306 """Test that we can store URIs in sets and as keys.""" 

307 uri1 = ButlerURI(TESTDIR) 

308 uri2 = uri1.join("test/") 

309 s = {uri1, uri2} 

310 self.assertIn(uri1, s) 

311 

312 d = {uri1: "1", uri2: "2"} 

313 self.assertEqual(d[uri2], "2") 

314 

315 def testWalk(self): 

316 """Test ButlerURI.walk().""" 

317 test_dir_uri = ButlerURI(TESTDIR) 

318 

319 file = test_dir_uri.join("config/basic/butler.yaml") 

320 found = list(ButlerURI.findFileResources([file])) 

321 self.assertEqual(found[0], file) 

322 

323 # Compare against the full local paths 

324 expected = set(p for p in glob.glob(os.path.join(TESTDIR, "config", "**"), recursive=True) 

325 if os.path.isfile(p)) 

326 found = set(u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")])) 

327 self.assertEqual(found, expected) 

328 

329 # Now solely the YAML files 

330 expected_yaml = set(glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"), recursive=True)) 

331 found = set(u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")], 

332 file_filter=r".*\.yaml$")) 

333 self.assertEqual(found, expected_yaml) 

334 

335 # Now two explicit directories and a file 

336 expected = set(glob.glob(os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"), recursive=True)) 

337 expected.update(set(glob.glob(os.path.join(TESTDIR, "config", "**", "templates", "*.yaml"), 

338 recursive=True))) 

339 expected.add(file.ospath) 

340 

341 found = set(u.ospath for u in ButlerURI.findFileResources([file, test_dir_uri.join("config/basic"), 

342 test_dir_uri.join("config/templates")], 

343 file_filter=r".*\.yaml$")) 

344 self.assertEqual(found, expected) 

345 

346 # Group by directory -- find everything and compare it with what 

347 # we expected to be there in total. We expect to find 9 directories 

348 # containing yaml files so make sure we only iterate 9 times. 

349 found_yaml = set() 

350 counter = 0 

351 for uris in ButlerURI.findFileResources([file, test_dir_uri.join("config/")], 

352 file_filter=r".*\.yaml$", grouped=True): 

353 found = set(u.ospath for u in uris) 

354 if found: 

355 counter += 1 

356 

357 found_yaml.update(found) 

358 

359 self.assertEqual(found_yaml, expected_yaml) 

360 self.assertEqual(counter, 9) 

361 

362 # Grouping but check that single files are returned in a single group 

363 # at the end 

364 file2 = test_dir_uri.join("config/templates/templates-bad.yaml") 

365 found = list(ButlerURI.findFileResources([file, file2, test_dir_uri.join("config/dbAuth")], 

366 grouped=True)) 

367 self.assertEqual(len(found), 2) 

368 self.assertEqual(list(found[1]), [file, file2]) 

369 

370 with self.assertRaises(ValueError): 

371 list(file.walk()) 

372 

373 def testRootURI(self): 

374 """Test ButlerURI.root_uri().""" 

375 uri = ButlerURI("https://www.notexist.com:8080/file/test") 

376 uri2 = ButlerURI("s3://www.notexist.com/file/test") 

377 self.assertEqual(uri.root_uri().geturl(), "https://www.notexist.com:8080/") 

378 self.assertEqual(uri2.root_uri().geturl(), "s3://www.notexist.com/") 

379 

380 def testJoin(self): 

381 """Test .join method.""" 

382 

383 root_str = "s3://bucket/hsc/payload/" 

384 root = ButlerURI(root_str) 

385 

386 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

387 add_dir = root.join("b/c/d/") 

388 self.assertTrue(add_dir.isdir()) 

389 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

390 

391 quote_example = "b&c.t@x#t" 

392 needs_quote = root.join(quote_example) 

393 self.assertEqual(needs_quote.unquoted_path, f"/hsc/payload/{quote_example}") 

394 

395 other = ButlerURI("file://localhost/test.txt") 

396 self.assertEqual(root.join(other), other) 

397 self.assertEqual(other.join("b/new.txt").geturl(), "file://localhost/b/new.txt") 

398 

399 joined = ButlerURI("s3://bucket/hsc/payload/").join(ButlerURI("test.qgraph", forceAbsolute=False)) 

400 self.assertEqual(joined, ButlerURI("s3://bucket/hsc/payload/test.qgraph")) 

401 

402 with self.assertRaises(ValueError): 

403 ButlerURI("s3://bucket/hsc/payload/").join(ButlerURI("test.qgraph")) 

404 

405 

406@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

407@mock_s3 

408class S3URITestCase(unittest.TestCase): 

409 """Tests involving S3""" 

410 

411 bucketName = "any_bucket" 

412 """Bucket name to use in tests""" 

413 

414 def setUp(self): 

415 # Local test directory 

416 self.tmpdir = makeTestTempDir(TESTDIR) 

417 

418 # set up some fake credentials if they do not exist 

419 self.usingDummyCredentials = setAwsEnvCredentials() 

420 

421 # MOTO needs to know that we expect Bucket bucketname to exist 

422 s3 = boto3.resource("s3") 

423 s3.create_bucket(Bucket=self.bucketName) 

424 

425 def tearDown(self): 

426 s3 = boto3.resource("s3") 

427 bucket = s3.Bucket(self.bucketName) 

428 try: 

429 bucket.objects.all().delete() 

430 except botocore.exceptions.ClientError as e: 

431 if e.response["Error"]["Code"] == "404": 

432 # the key was not reachable - pass 

433 pass 

434 else: 

435 raise 

436 

437 bucket = s3.Bucket(self.bucketName) 

438 bucket.delete() 

439 

440 # unset any potentially set dummy credentials 

441 if self.usingDummyCredentials: 

442 unsetAwsEnvCredentials() 

443 

444 shutil.rmtree(self.tmpdir, ignore_errors=True) 

445 

446 def makeS3Uri(self, path): 

447 return f"s3://{self.bucketName}/{path}" 

448 

449 def testTransfer(self): 

450 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

451 content = "Content is some content\nwith something to say\n\n" 

452 src.write(content.encode()) 

453 self.assertTrue(src.exists()) 

454 self.assertEqual(src.size(), len(content.encode())) 

455 

456 dest = ButlerURI(self.makeS3Uri("test.txt")) 

457 self.assertFalse(dest.exists()) 

458 

459 with self.assertRaises(FileNotFoundError): 

460 dest.size() 

461 

462 dest.transfer_from(src, transfer="copy") 

463 self.assertTrue(dest.exists()) 

464 

465 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

466 dest2.transfer_from(dest, transfer="copy") 

467 self.assertTrue(dest2.exists()) 

468 

469 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

470 local.transfer_from(dest2, transfer="copy") 

471 with open(local.ospath, "r") as fd: 

472 new_content = fd.read() 

473 self.assertEqual(new_content, content) 

474 

475 with self.assertRaises(ValueError): 

476 dest2.transfer_from(local, transfer="symlink") 

477 

478 b = dest.read() 

479 self.assertEqual(b.decode(), new_content) 

480 

481 nbytes = 10 

482 subset = dest.read(size=nbytes) 

483 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

484 self.assertEqual(subset.decode(), content[:nbytes]) 

485 

486 with self.assertRaises(FileExistsError): 

487 dest.transfer_from(src, transfer="copy") 

488 

489 dest.transfer_from(src, transfer="copy", overwrite=True) 

490 

491 def testWalk(self): 

492 """Test that we can list an S3 bucket""" 

493 # Files we want to create 

494 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json") 

495 expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected] 

496 for uri in expected_uris: 

497 # Doesn't matter what we write 

498 uri.write("123".encode()) 

499 

500 # Find all the files in the a/ tree 

501 found = set(uri.path for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))])) 

502 self.assertEqual(found, {uri.path for uri in expected_uris}) 

503 

504 # Find all the files in the a/ tree but group by folder 

505 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

506 grouped=True) 

507 expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt",), ("/a/b/c/d/v.json",)) 

508 

509 for got, expect in zip(found, expected): 

510 self.assertEqual(tuple(u.path for u in got), expect) 

511 

512 # Find only JSON files 

513 found = set(uri.path for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

514 file_filter=r"\.json$")) 

515 self.assertEqual(found, {uri.path for uri in expected_uris if uri.path.endswith(".json")}) 

516 

517 # JSON files grouped by directory 

518 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

519 file_filter=r"\.json$", grouped=True) 

520 expected = (("/a/z.json",), ("/a/b/c/d/v.json",)) 

521 

522 for got, expect in zip(found, expected): 

523 self.assertEqual(tuple(u.path for u in got), expect) 

524 

525 # Check pagination works with large numbers of files. S3 API limits 

526 # us to 1000 response per list_objects call so create lots of files 

527 created = set() 

528 counter = 1 

529 n_dir1 = 1100 

530 while counter <= n_dir1: 

531 new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt")) 

532 new.write(f"{counter}".encode()) 

533 created.add(str(new)) 

534 counter += 1 

535 counter = 1 

536 # Put some in a subdirectory to make sure we are looking in a 

537 # hierarchy. 

538 n_dir2 = 100 

539 while counter <= n_dir2: 

540 new = ButlerURI(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt")) 

541 new.write(f"{counter}".encode()) 

542 created.add(str(new)) 

543 counter += 1 

544 

545 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))]) 

546 self.assertEqual({str(u) for u in found}, created) 

547 

548 # Again with grouping. 

549 found = list(ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))], grouped=True)) 

550 self.assertEqual(len(found), 2) 

551 dir_1 = list(found[0]) 

552 dir_2 = list(found[1]) 

553 self.assertEqual(len(dir_1), n_dir1) 

554 self.assertEqual(len(dir_2), n_dir2) 

555 

556 def testWrite(self): 

557 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

558 content = "abcdefghijklmnopqrstuv\n" 

559 s3write.write(content.encode()) 

560 self.assertEqual(s3write.read().decode(), content) 

561 

562 def testRelative(self): 

563 """Check that we can get subpaths back from two URIs""" 

564 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

565 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

566 

567 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

568 

569 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

570 self.assertFalse(not_child.relative_to(parent)) 

571 

572 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

573 self.assertFalse(child.relative_to(not_s3)) 

574 

575 def testQuoting(self): 

576 """Check that quoting works.""" 

577 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

578 subpath = "rootdir/dir1+/file?.txt" 

579 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

580 

581 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

582 self.assertEqual(child.basename(), "file?.txt") 

583 self.assertEqual(child.relativeToPathRoot, subpath) 

584 self.assertIn("%", child.path) 

585 self.assertEqual(child.unquoted_path, "/" + subpath) 

586 

587 

588# Mock required environment variables during tests 

589@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

590 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join( 

591 TESTDIR, "config/testConfigs/webdav/token"), 

592 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"}) 

593class WebdavURITestCase(unittest.TestCase): 

594 

595 def setUp(self): 

596 serverRoot = "www.not-exists.orgx" 

597 existingFolderName = "existingFolder" 

598 existingFileName = "existingFile" 

599 notExistingFileName = "notExistingFile" 

600 

601 self.baseURL = ButlerURI( 

602 f"https://{serverRoot}", forceDirectory=True) 

603 self.existingFileButlerURI = ButlerURI( 

604 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

605 self.notExistingFileButlerURI = ButlerURI( 

606 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

607 self.existingFolderButlerURI = ButlerURI( 

608 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

609 self.notExistingFolderButlerURI = ButlerURI( 

610 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

611 

612 # Need to declare the options 

613 responses.add(responses.OPTIONS, 

614 self.baseURL.geturl(), 

615 status=200, headers={"DAV": "1,2,3"}) 

616 

617 # Used by ButlerHttpURI.exists() 

618 responses.add(responses.HEAD, 

619 self.existingFileButlerURI.geturl(), 

620 status=200, headers={'Content-Length': '1024'}) 

621 responses.add(responses.HEAD, 

622 self.notExistingFileButlerURI.geturl(), 

623 status=404) 

624 

625 # Used by ButlerHttpURI.read() 

626 responses.add(responses.GET, 

627 self.existingFileButlerURI.geturl(), 

628 status=200, 

629 body=str.encode("It works!")) 

630 responses.add(responses.GET, 

631 self.notExistingFileButlerURI.geturl(), 

632 status=404) 

633 

634 # Used by ButlerHttpURI.write() 

635 responses.add(responses.PUT, 

636 self.existingFileButlerURI.geturl(), 

637 status=201) 

638 

639 # Used by ButlerHttpURI.transfer_from() 

640 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

641 method="COPY", 

642 headers={"Destination": self.existingFileButlerURI.geturl()}, 

643 status=201)) 

644 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

645 method="COPY", 

646 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

647 status=201)) 

648 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

649 method="MOVE", 

650 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

651 status=201)) 

652 

653 # Used by ButlerHttpURI.remove() 

654 responses.add(responses.DELETE, 

655 self.existingFileButlerURI.geturl(), 

656 status=200) 

657 responses.add(responses.DELETE, 

658 self.notExistingFileButlerURI.geturl(), 

659 status=404) 

660 

661 # Used by ButlerHttpURI.mkdir() 

662 responses.add(responses.HEAD, 

663 self.existingFolderButlerURI.geturl(), 

664 status=200, headers={'Content-Length': '1024'}) 

665 responses.add(responses.HEAD, 

666 self.baseURL.geturl(), 

667 status=200, headers={'Content-Length': '1024'}) 

668 responses.add(responses.HEAD, 

669 self.notExistingFolderButlerURI.geturl(), 

670 status=404) 

671 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

672 method="MKCOL", 

673 status=201)) 

674 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

675 method="MKCOL", 

676 status=403)) 

677 

678 @responses.activate 

679 def testExists(self): 

680 

681 self.assertTrue(self.existingFileButlerURI.exists()) 

682 self.assertFalse(self.notExistingFileButlerURI.exists()) 

683 

684 self.assertEqual(self.existingFileButlerURI.size(), 1024) 

685 with self.assertRaises(FileNotFoundError): 

686 self.notExistingFileButlerURI.size() 

687 

688 @responses.activate 

689 def testRemove(self): 

690 

691 self.assertIsNone(self.existingFileButlerURI.remove()) 

692 with self.assertRaises(FileNotFoundError): 

693 self.notExistingFileButlerURI.remove() 

694 

695 @responses.activate 

696 def testMkdir(self): 

697 

698 # The mock means that we can't check this now exists 

699 self.notExistingFolderButlerURI.mkdir() 

700 

701 # This should do nothing 

702 self.existingFolderButlerURI.mkdir() 

703 

704 with self.assertRaises(ValueError): 

705 self.notExistingFileButlerURI.mkdir() 

706 

707 @responses.activate 

708 def testRead(self): 

709 

710 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

711 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

712 with self.assertRaises(FileNotFoundError): 

713 self.notExistingFileButlerURI.read() 

714 

715 @responses.activate 

716 def testWrite(self): 

717 

718 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

719 with self.assertRaises(FileExistsError): 

720 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

721 

722 @responses.activate 

723 def testTransfer(self): 

724 

725 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

726 src=self.existingFileButlerURI)) 

727 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

728 src=self.existingFileButlerURI, 

729 transfer="move")) 

730 with self.assertRaises(FileExistsError): 

731 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

732 with self.assertRaises(ValueError): 

733 self.notExistingFileButlerURI.transfer_from( 

734 src=self.existingFileButlerURI, 

735 transfer="unsupported") 

736 

737 def testParent(self): 

738 

739 self.assertEqual(self.existingFolderButlerURI.geturl(), 

740 self.notExistingFileButlerURI.parent().geturl()) 

741 self.assertEqual(self.baseURL.geturl(), 

742 self.baseURL.parent().geturl()) 

743 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

744 self.existingFileButlerURI.dirname().geturl()) 

745 

746 

747if __name__ == "__main__": 747 ↛ 748line 747 didn't jump to line 748, because the condition on line 747 was never true

748 unittest.main()