Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import glob 

23import os 

24import shutil 

25import unittest 

26import urllib.parse 

27import responses 

28 

29try: 

30 import boto3 

31 import botocore 

32 from moto import mock_s3 

33except ImportError: 

34 boto3 = None 

35 

36 def mock_s3(cls): 

37 """A no-op decorator in case moto mock_s3 can not be imported. 

38 """ 

39 return cls 

40 

41from lsst.daf.butler import ButlerURI 

42from lsst.daf.butler.core._butlerUri.s3utils import (setAwsEnvCredentials, 

43 unsetAwsEnvCredentials) 

44from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

45 

46TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

47 

48 

49class FileURITestCase(unittest.TestCase): 

50 """Concrete tests for local files""" 

51 

52 def setUp(self): 

53 # Use a local tempdir because on macOS the temp dirs use symlinks 

54 # so relsymlink gets quite confused. 

55 self.tmpdir = makeTestTempDir(TESTDIR) 

56 

57 def tearDown(self): 

58 removeTestTempDir(self.tmpdir) 

59 

60 def testFile(self): 

61 file = os.path.join(self.tmpdir, "test.txt") 

62 uri = ButlerURI(file) 

63 self.assertFalse(uri.exists(), f"{uri} should not exist") 

64 self.assertEqual(uri.ospath, file) 

65 

66 content = "abcdefghijklmnopqrstuv\n" 

67 uri.write(content.encode()) 

68 self.assertTrue(os.path.exists(file), "File should exist locally") 

69 self.assertTrue(uri.exists(), f"{uri} should now exist") 

70 self.assertEqual(uri.read().decode(), content) 

71 

72 # Check that creating a URI from a URI returns the same thing 

73 uri2 = ButlerURI(uri) 

74 self.assertEqual(uri, uri2) 

75 self.assertEqual(id(uri), id(uri2)) 

76 

77 def testExtension(self): 

78 file = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

79 self.assertEqual(file.updatedExtension(None), file) 

80 self.assertEqual(file.updatedExtension(".txt"), file) 

81 self.assertEqual(id(file.updatedExtension(".txt")), id(file)) 

82 

83 fits = file.updatedExtension(".fits.gz") 

84 self.assertEqual(fits.basename(), "test.fits.gz") 

85 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

86 

87 def testRelative(self): 

88 """Check that we can get subpaths back from two URIs""" 

89 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

90 self.assertTrue(parent.isdir()) 

91 child = ButlerURI(os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True) 

92 

93 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

94 

95 not_child = ButlerURI("/a/b/dir1/file.txt") 

96 self.assertFalse(not_child.relative_to(parent)) 

97 self.assertFalse(not_child.isdir()) 

98 

99 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

100 self.assertFalse(child.relative_to(not_directory)) 

101 

102 # Relative URIs 

103 parent = ButlerURI("a/b/", forceAbsolute=False) 

104 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

105 self.assertFalse(child.scheme) 

106 self.assertEqual(child.relative_to(parent), "c/d.txt") 

107 

108 # File URI and schemeless URI 

109 parent = ButlerURI("file:/a/b/c/") 

110 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

111 

112 # If the child is relative and the parent is absolute we assume 

113 # that the child is a child of the parent unless it uses ".." 

114 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

115 

116 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

117 self.assertFalse(child.relative_to(parent)) 

118 

119 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

120 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

121 

122 def testParents(self): 

123 """Test of splitting and parent walking.""" 

124 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

125 child_file = parent.join("subdir/file.txt") 

126 self.assertFalse(child_file.isdir()) 

127 child_subdir, file = child_file.split() 

128 self.assertEqual(file, "file.txt") 

129 self.assertTrue(child_subdir.isdir()) 

130 self.assertEqual(child_file.dirname(), child_subdir) 

131 self.assertEqual(child_file.basename(), file) 

132 self.assertEqual(child_file.parent(), child_subdir) 

133 derived_parent = child_subdir.parent() 

134 self.assertEqual(derived_parent, parent) 

135 self.assertTrue(derived_parent.isdir()) 

136 self.assertEqual(child_file.parent().parent(), parent) 

137 

138 def testEnvVar(self): 

139 """Test that environment variables are expanded.""" 

140 

141 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

142 uri = ButlerURI("${MY_TEST_DIR}/d.txt") 

143 self.assertEqual(uri.path, "/a/b/c/d.txt") 

144 self.assertEqual(uri.scheme, "file") 

145 

146 # This will not expand 

147 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

148 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

149 self.assertFalse(uri.scheme) 

150 

151 def testMkdir(self): 

152 tmpdir = ButlerURI(self.tmpdir) 

153 newdir = tmpdir.join("newdir/seconddir") 

154 newdir.mkdir() 

155 self.assertTrue(newdir.exists()) 

156 newfile = newdir.join("temp.txt") 

157 newfile.write("Data".encode()) 

158 self.assertTrue(newfile.exists()) 

159 

160 def testTransfer(self): 

161 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

162 content = "Content is some content\nwith something to say\n\n" 

163 src.write(content.encode()) 

164 

165 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

166 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

167 dest.transfer_from(src, transfer=mode) 

168 self.assertTrue(dest.exists(), f"Check that {dest} exists (transfer={mode})") 

169 

170 with open(dest.ospath, "r") as fh: 

171 new_content = fh.read() 

172 self.assertEqual(new_content, content) 

173 

174 if mode in ("symlink", "relsymlink"): 

175 self.assertTrue(os.path.islink(dest.ospath), f"Check that {dest} is symlink") 

176 

177 with self.assertRaises(FileExistsError): 

178 dest.transfer_from(src, transfer=mode) 

179 

180 dest.transfer_from(src, transfer=mode, overwrite=True) 

181 

182 os.remove(dest.ospath) 

183 

184 b = src.read() 

185 self.assertEqual(b.decode(), new_content) 

186 

187 nbytes = 10 

188 subset = src.read(size=nbytes) 

189 self.assertEqual(len(subset), nbytes) 

190 self.assertEqual(subset.decode(), content[:nbytes]) 

191 

192 with self.assertRaises(ValueError): 

193 src.transfer_from(src, transfer="unknown") 

194 

195 def testResource(self): 

196 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

197 self.assertTrue(u.exists(), f"Check {u} exists") 

198 

199 content = u.read().decode() 

200 self.assertTrue(content.startswith("datastore:")) 

201 

202 truncated = u.read(size=9).decode() 

203 self.assertEqual(truncated, "datastore") 

204 

205 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

206 self.assertTrue(u.exists(), f"Check directory {d} exists") 

207 

208 j = d.join("datastore.yaml") 

209 self.assertEqual(u, j) 

210 self.assertFalse(j.dirLike) 

211 self.assertFalse(j.isdir()) 

212 self.assertFalse(d.join("not-there.yaml").exists()) 

213 

214 def testEscapes(self): 

215 """Special characters in file paths""" 

216 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

217 self.assertFalse(src.scheme) 

218 src.write(b"Some content") 

219 self.assertTrue(src.exists()) 

220 

221 # Use the internal API to force to a file 

222 file = src._force_to_file() 

223 self.assertTrue(file.exists()) 

224 self.assertIn("???", file.ospath) 

225 self.assertNotIn("???", file.path) 

226 

227 file = file.updatedFile("tests??.txt") 

228 self.assertNotIn("??.txt", file.path) 

229 file.write(b"Other content") 

230 self.assertEqual(file.read(), b"Other content") 

231 

232 src = src.updatedFile("tests??.txt") 

233 self.assertIn("??.txt", src.path) 

234 self.assertEqual(file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}") 

235 

236 # File URI and schemeless URI 

237 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

238 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

239 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

240 

241 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

242 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

243 

244 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

245 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

246 

247 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

248 

249 # Schemeless so should not quote 

250 dir = ButlerURI("bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True) 

251 self.assertIn("???", dir.ospath) 

252 self.assertIn("???", dir.path) 

253 self.assertFalse(dir.scheme) 

254 

255 # dir.join() morphs into a file scheme 

256 new = dir.join("test_j.txt") 

257 self.assertIn("???", new.ospath, f"Checking {new}") 

258 new.write(b"Content") 

259 

260 new2name = "###/test??.txt" 

261 new2 = dir.join(new2name) 

262 self.assertIn("???", new2.ospath) 

263 new2.write(b"Content") 

264 self.assertTrue(new2.ospath.endswith(new2name)) 

265 self.assertEqual(new.read(), new2.read()) 

266 

267 fdir = dir._force_to_file() 

268 self.assertNotIn("???", fdir.path) 

269 self.assertIn("???", fdir.ospath) 

270 self.assertEqual(fdir.scheme, "file") 

271 fnew = dir.join("test_jf.txt") 

272 fnew.write(b"Content") 

273 

274 fnew2 = fdir.join(new2name) 

275 fnew2.write(b"Content") 

276 self.assertTrue(fnew2.ospath.endswith(new2name)) 

277 self.assertNotIn("###", fnew2.path) 

278 

279 self.assertEqual(fnew.read(), fnew2.read()) 

280 

281 # Test that children relative to schemeless and file schemes 

282 # still return the same unquoted name 

283 self.assertEqual(fnew2.relative_to(fdir), new2name) 

284 self.assertEqual(fnew2.relative_to(dir), new2name) 

285 self.assertEqual(new2.relative_to(fdir), new2name, f"{new2} vs {fdir}") 

286 self.assertEqual(new2.relative_to(dir), new2name) 

287 

288 # Check for double quoting 

289 plus_path = "/a/b/c+d/" 

290 with self.assertLogs(level="WARNING"): 

291 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

292 self.assertEqual(uri.ospath, plus_path) 

293 

294 # Check that # is not escaped for schemeless URIs 

295 hash_path = "/a/b#/c&d#xyz" 

296 hpos = hash_path.rfind("#") 

297 uri = ButlerURI(hash_path) 

298 self.assertEqual(uri.ospath, hash_path[:hpos]) 

299 self.assertEqual(uri.fragment, hash_path[hpos + 1:]) 

300 

301 def testHash(self): 

302 """Test that we can store URIs in sets and as keys.""" 

303 uri1 = ButlerURI(TESTDIR) 

304 uri2 = uri1.join("test/") 

305 s = {uri1, uri2} 

306 self.assertIn(uri1, s) 

307 

308 d = {uri1: "1", uri2: "2"} 

309 self.assertEqual(d[uri2], "2") 

310 

311 def testWalk(self): 

312 """Test ButlerURI.walk().""" 

313 test_dir_uri = ButlerURI(TESTDIR) 

314 

315 file = test_dir_uri.join("config/basic/butler.yaml") 

316 found = list(ButlerURI.findFileResources([file])) 

317 self.assertEqual(found[0], file) 

318 

319 # Compare against the full local paths 

320 expected = set(p for p in glob.glob(os.path.join(TESTDIR, "config", "**"), recursive=True) 

321 if os.path.isfile(p)) 

322 found = set(u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")])) 

323 self.assertEqual(found, expected) 

324 

325 # Now solely the YAML files 

326 expected_yaml = set(glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"), recursive=True)) 

327 found = set(u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")], 

328 file_filter=r".*\.yaml$")) 

329 self.assertEqual(found, expected_yaml) 

330 

331 # Now two explicit directories and a file 

332 expected = set(glob.glob(os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"), recursive=True)) 

333 expected.update(set(glob.glob(os.path.join(TESTDIR, "config", "**", "templates", "*.yaml"), 

334 recursive=True))) 

335 expected.add(file.ospath) 

336 

337 found = set(u.ospath for u in ButlerURI.findFileResources([file, test_dir_uri.join("config/basic"), 

338 test_dir_uri.join("config/templates")], 

339 file_filter=r".*\.yaml$")) 

340 self.assertEqual(found, expected) 

341 

342 # Group by directory -- find everything and compare it with what 

343 # we expected to be there in total. We expect to find 9 directories 

344 # containing yaml files so make sure we only iterate 9 times. 

345 found_yaml = set() 

346 counter = 0 

347 for uris in ButlerURI.findFileResources([file, test_dir_uri.join("config/")], 

348 file_filter=r".*\.yaml$", grouped=True): 

349 found = set(u.ospath for u in uris) 

350 if found: 

351 counter += 1 

352 

353 found_yaml.update(found) 

354 

355 self.assertEqual(found_yaml, expected_yaml) 

356 self.assertEqual(counter, 9) 

357 

358 # Grouping but check that single files are returned in a single group 

359 # at the end 

360 file2 = test_dir_uri.join("config/templates/templates-bad.yaml") 

361 found = list(ButlerURI.findFileResources([file, file2, test_dir_uri.join("config/dbAuth")], 

362 grouped=True)) 

363 self.assertEqual(len(found), 2) 

364 self.assertEqual(list(found[1]), [file, file2]) 

365 

366 with self.assertRaises(ValueError): 

367 list(file.walk()) 

368 

369 

370@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

371@mock_s3 

372class S3URITestCase(unittest.TestCase): 

373 """Tests involving S3""" 

374 

375 bucketName = "any_bucket" 

376 """Bucket name to use in tests""" 

377 

378 def setUp(self): 

379 # Local test directory 

380 self.tmpdir = makeTestTempDir(TESTDIR) 

381 

382 # set up some fake credentials if they do not exist 

383 self.usingDummyCredentials = setAwsEnvCredentials() 

384 

385 # MOTO needs to know that we expect Bucket bucketname to exist 

386 s3 = boto3.resource("s3") 

387 s3.create_bucket(Bucket=self.bucketName) 

388 

389 def tearDown(self): 

390 s3 = boto3.resource("s3") 

391 bucket = s3.Bucket(self.bucketName) 

392 try: 

393 bucket.objects.all().delete() 

394 except botocore.exceptions.ClientError as e: 

395 if e.response["Error"]["Code"] == "404": 

396 # the key was not reachable - pass 

397 pass 

398 else: 

399 raise 

400 

401 bucket = s3.Bucket(self.bucketName) 

402 bucket.delete() 

403 

404 # unset any potentially set dummy credentials 

405 if self.usingDummyCredentials: 

406 unsetAwsEnvCredentials() 

407 

408 shutil.rmtree(self.tmpdir, ignore_errors=True) 

409 

410 def makeS3Uri(self, path): 

411 return f"s3://{self.bucketName}/{path}" 

412 

413 def testTransfer(self): 

414 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

415 content = "Content is some content\nwith something to say\n\n" 

416 src.write(content.encode()) 

417 

418 dest = ButlerURI(self.makeS3Uri("test.txt")) 

419 self.assertFalse(dest.exists()) 

420 dest.transfer_from(src, transfer="copy") 

421 self.assertTrue(dest.exists()) 

422 

423 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

424 dest2.transfer_from(dest, transfer="copy") 

425 self.assertTrue(dest2.exists()) 

426 

427 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

428 local.transfer_from(dest2, transfer="copy") 

429 with open(local.ospath, "r") as fd: 

430 new_content = fd.read() 

431 self.assertEqual(new_content, content) 

432 

433 with self.assertRaises(ValueError): 

434 dest2.transfer_from(local, transfer="symlink") 

435 

436 b = dest.read() 

437 self.assertEqual(b.decode(), new_content) 

438 

439 nbytes = 10 

440 subset = dest.read(size=nbytes) 

441 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

442 self.assertEqual(subset.decode(), content[:nbytes]) 

443 

444 with self.assertRaises(FileExistsError): 

445 dest.transfer_from(src, transfer="copy") 

446 

447 dest.transfer_from(src, transfer="copy", overwrite=True) 

448 

449 def testWalk(self): 

450 """Test that we can list an S3 bucket""" 

451 # Files we want to create 

452 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json") 

453 expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected] 

454 for uri in expected_uris: 

455 # Doesn't matter what we write 

456 uri.write("123".encode()) 

457 

458 # Find all the files in the a/ tree 

459 found = set(uri.path for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))])) 

460 self.assertEqual(found, {uri.path for uri in expected_uris}) 

461 

462 # Find all the files in the a/ tree but group by folder 

463 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

464 grouped=True) 

465 expected = (("/a/x.txt", "/a/y.txt", "/a/z.json"), ("/a/b/w.txt",), ("/a/b/c/d/v.json",)) 

466 

467 for got, expect in zip(found, expected): 

468 self.assertEqual(tuple(u.path for u in got), expect) 

469 

470 # Find only JSON files 

471 found = set(uri.path for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

472 file_filter=r"\.json$")) 

473 self.assertEqual(found, {uri.path for uri in expected_uris if uri.path.endswith(".json")}) 

474 

475 # JSON files grouped by directory 

476 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))], 

477 file_filter=r"\.json$", grouped=True) 

478 expected = (("/a/z.json",), ("/a/b/c/d/v.json",)) 

479 

480 for got, expect in zip(found, expected): 

481 self.assertEqual(tuple(u.path for u in got), expect) 

482 

483 # Check pagination works with large numbers of files. S3 API limits 

484 # us to 1000 response per list_objects call so create lots of files 

485 created = set() 

486 counter = 1 

487 n_dir1 = 1100 

488 while counter <= n_dir1: 

489 new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt")) 

490 new.write(f"{counter}".encode()) 

491 created.add(str(new)) 

492 counter += 1 

493 counter = 1 

494 # Put some in a subdirectory to make sure we are looking in a 

495 # hierarchy. 

496 n_dir2 = 100 

497 while counter <= n_dir2: 

498 new = ButlerURI(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt")) 

499 new.write(f"{counter}".encode()) 

500 created.add(str(new)) 

501 counter += 1 

502 

503 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))]) 

504 self.assertEqual({str(u) for u in found}, created) 

505 

506 # Again with grouping. 

507 found = list(ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))], grouped=True)) 

508 self.assertEqual(len(found), 2) 

509 dir_1 = list(found[0]) 

510 dir_2 = list(found[1]) 

511 self.assertEqual(len(dir_1), n_dir1) 

512 self.assertEqual(len(dir_2), n_dir2) 

513 

514 def testWrite(self): 

515 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

516 content = "abcdefghijklmnopqrstuv\n" 

517 s3write.write(content.encode()) 

518 self.assertEqual(s3write.read().decode(), content) 

519 

520 def testRelative(self): 

521 """Check that we can get subpaths back from two URIs""" 

522 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

523 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

524 

525 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

526 

527 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

528 self.assertFalse(not_child.relative_to(parent)) 

529 

530 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

531 self.assertFalse(child.relative_to(not_s3)) 

532 

533 def testQuoting(self): 

534 """Check that quoting works.""" 

535 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

536 subpath = "rootdir/dir1+/file?.txt" 

537 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

538 

539 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

540 self.assertEqual(child.basename(), "file?.txt") 

541 self.assertEqual(child.relativeToPathRoot, subpath) 

542 self.assertIn("%", child.path) 

543 self.assertEqual(child.unquoted_path, "/" + subpath) 

544 

545 

546# Mock required environment variables during tests 

547@unittest.mock.patch.dict(os.environ, {"LSST_BUTLER_WEBDAV_AUTH": "TOKEN", 

548 "LSST_BUTLER_WEBDAV_TOKEN_FILE": os.path.join( 

549 TESTDIR, "config/testConfigs/webdav/token"), 

550 "LSST_BUTLER_WEBDAV_CA_BUNDLE": "/path/to/ca/certs"}) 

551class WebdavURITestCase(unittest.TestCase): 

552 

553 def setUp(self): 

554 serverRoot = "www.not-exists.orgx" 

555 existingFolderName = "existingFolder" 

556 existingFileName = "existingFile" 

557 notExistingFileName = "notExistingFile" 

558 

559 self.baseURL = ButlerURI( 

560 f"https://{serverRoot}", forceDirectory=True) 

561 self.existingFileButlerURI = ButlerURI( 

562 f"https://{serverRoot}/{existingFolderName}/{existingFileName}") 

563 self.notExistingFileButlerURI = ButlerURI( 

564 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}") 

565 self.existingFolderButlerURI = ButlerURI( 

566 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True) 

567 self.notExistingFolderButlerURI = ButlerURI( 

568 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True) 

569 

570 # Need to declare the options 

571 responses.add(responses.OPTIONS, 

572 self.baseURL.geturl(), 

573 status=200, headers={"DAV": "1,2,3"}) 

574 

575 # Used by ButlerHttpURI.exists() 

576 responses.add(responses.HEAD, 

577 self.existingFileButlerURI.geturl(), 

578 status=200, headers={'Content-Length': '1024'}) 

579 responses.add(responses.HEAD, 

580 self.notExistingFileButlerURI.geturl(), 

581 status=404) 

582 

583 # Used by ButlerHttpURI.read() 

584 responses.add(responses.GET, 

585 self.existingFileButlerURI.geturl(), 

586 status=200, 

587 body=str.encode("It works!")) 

588 responses.add(responses.GET, 

589 self.notExistingFileButlerURI.geturl(), 

590 status=404) 

591 

592 # Used by ButlerHttpURI.write() 

593 responses.add(responses.PUT, 

594 self.existingFileButlerURI.geturl(), 

595 status=201) 

596 

597 # Used by ButlerHttpURI.transfer_from() 

598 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

599 method="COPY", 

600 headers={"Destination": self.existingFileButlerURI.geturl()}, 

601 status=201)) 

602 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

603 method="COPY", 

604 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

605 status=201)) 

606 responses.add(responses.Response(url=self.existingFileButlerURI.geturl(), 

607 method="MOVE", 

608 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

609 status=201)) 

610 

611 # Used by ButlerHttpURI.remove() 

612 responses.add(responses.DELETE, 

613 self.existingFileButlerURI.geturl(), 

614 status=200) 

615 responses.add(responses.DELETE, 

616 self.notExistingFileButlerURI.geturl(), 

617 status=404) 

618 

619 # Used by ButlerHttpURI.mkdir() 

620 responses.add(responses.HEAD, 

621 self.existingFolderButlerURI.geturl(), 

622 status=200, headers={'Content-Length': '1024'}) 

623 responses.add(responses.HEAD, 

624 self.baseURL.geturl(), 

625 status=200, headers={'Content-Length': '1024'}) 

626 responses.add(responses.HEAD, 

627 self.notExistingFolderButlerURI.geturl(), 

628 status=404) 

629 responses.add(responses.Response(url=self.notExistingFolderButlerURI.geturl(), 

630 method="MKCOL", 

631 status=201)) 

632 responses.add(responses.Response(url=self.existingFolderButlerURI.geturl(), 

633 method="MKCOL", 

634 status=403)) 

635 

636 @responses.activate 

637 def testExists(self): 

638 

639 self.assertTrue(self.existingFileButlerURI.exists()) 

640 self.assertFalse(self.notExistingFileButlerURI.exists()) 

641 

642 @responses.activate 

643 def testRemove(self): 

644 

645 self.assertIsNone(self.existingFileButlerURI.remove()) 

646 with self.assertRaises(FileNotFoundError): 

647 self.notExistingFileButlerURI.remove() 

648 

649 @responses.activate 

650 def testMkdir(self): 

651 

652 # The mock means that we can't check this now exists 

653 self.notExistingFolderButlerURI.mkdir() 

654 

655 # This should do nothing 

656 self.existingFolderButlerURI.mkdir() 

657 

658 with self.assertRaises(ValueError): 

659 self.notExistingFileButlerURI.mkdir() 

660 

661 @responses.activate 

662 def testRead(self): 

663 

664 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

665 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

666 with self.assertRaises(FileNotFoundError): 

667 self.notExistingFileButlerURI.read() 

668 

669 @responses.activate 

670 def testWrite(self): 

671 

672 self.assertIsNone(self.existingFileButlerURI.write(data=str.encode("Some content."))) 

673 with self.assertRaises(FileExistsError): 

674 self.existingFileButlerURI.write(data=str.encode("Some content."), overwrite=False) 

675 

676 @responses.activate 

677 def testTransfer(self): 

678 

679 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

680 src=self.existingFileButlerURI)) 

681 self.assertIsNone(self.notExistingFileButlerURI.transfer_from( 

682 src=self.existingFileButlerURI, 

683 transfer="move")) 

684 with self.assertRaises(FileExistsError): 

685 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

686 with self.assertRaises(ValueError): 

687 self.notExistingFileButlerURI.transfer_from( 

688 src=self.existingFileButlerURI, 

689 transfer="unsupported") 

690 

691 def testParent(self): 

692 

693 self.assertEqual(self.existingFolderButlerURI.geturl(), 

694 self.notExistingFileButlerURI.parent().geturl()) 

695 self.assertEqual(self.baseURL.geturl(), 

696 self.baseURL.parent().geturl()) 

697 self.assertEqual(self.existingFileButlerURI.parent().geturl(), 

698 self.existingFileButlerURI.dirname().geturl()) 

699 

700 

701if __name__ == "__main__": 701 ↛ 702line 701 didn't jump to line 702, because the condition on line 701 was never true

702 unittest.main()