Coverage for tests/test_uri.py: 13%

670 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 19:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import glob 

23import importlib 

24import os 

25import shutil 

26import stat 

27import tempfile 

28import unittest 

29import urllib.parse 

30 

31import requests 

32import responses 

33 

34try: 

35 import boto3 

36 import botocore 

37 from moto import mock_s3 

38except ImportError: 

39 boto3 = None 

40 

41 def mock_s3(cls): 

42 """A no-op decorator in case moto mock_s3 can not be imported.""" 

43 return cls 

44 

45 

46import lsst.daf.butler.core._butlerUri.http as httpUri 

47from lsst.daf.butler import ButlerURI 

48from lsst.daf.butler.core._butlerUri.s3utils import ( 

49 setAwsEnvCredentials, 

50 unsetAwsEnvCredentials, 

51) 

52from lsst.daf.butler.tests.utils import makeTestTempDir, removeTestTempDir 

53 

54TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

55 

56 

57class FileURITestCase(unittest.TestCase): 

58 """Concrete tests for local files""" 

59 

60 def setUp(self): 

61 # Use a local tempdir because on macOS the temp dirs use symlinks 

62 # so relsymlink gets quite confused. 

63 self.tmpdir = makeTestTempDir(TESTDIR) 

64 

65 def tearDown(self): 

66 removeTestTempDir(self.tmpdir) 

67 

68 def testFile(self): 

69 file = os.path.join(self.tmpdir, "test.txt") 

70 uri = ButlerURI(file) 

71 self.assertFalse(uri.exists(), f"{uri} should not exist") 

72 self.assertEqual(uri.ospath, file) 

73 

74 content = "abcdefghijklmnopqrstuv\n" 

75 uri.write(content.encode()) 

76 self.assertTrue(os.path.exists(file), "File should exist locally") 

77 self.assertTrue(uri.exists(), f"{uri} should now exist") 

78 self.assertEqual(uri.read().decode(), content) 

79 self.assertEqual(uri.size(), len(content.encode())) 

80 

81 with self.assertRaises(FileNotFoundError): 

82 ButlerURI("file/not/there.txt").size() 

83 

84 # Check that creating a URI from a URI returns the same thing 

85 uri2 = ButlerURI(uri) 

86 self.assertEqual(uri, uri2) 

87 self.assertEqual(id(uri), id(uri2)) 

88 

89 with self.assertRaises(ValueError): 

90 # Scheme-less URIs are not allowed to support non-file roots 

91 # at the present time. This may change in the future to become 

92 # equivalent to ButlerURI.join() 

93 ButlerURI("a/b.txt", root=ButlerURI("s3://bucket/a/b/")) 

94 

95 def testExtension(self): 

96 file = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

97 self.assertEqual(file.updatedExtension(None), file) 

98 self.assertEqual(file.updatedExtension(".txt"), file) 

99 self.assertEqual(id(file.updatedExtension(".txt")), id(file)) 

100 

101 fits = file.updatedExtension(".fits.gz") 

102 self.assertEqual(fits.basename(), "test.fits.gz") 

103 self.assertEqual(fits.updatedExtension(".jpeg").basename(), "test.jpeg") 

104 

105 def testRelative(self): 

106 """Check that we can get subpaths back from two URIs""" 

107 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

108 self.assertTrue(parent.isdir()) 

109 child = ButlerURI( 

110 os.path.join(self.tmpdir, "dir1", "file.txt"), forceAbsolute=True 

111 ) 

112 

113 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

114 

115 not_child = ButlerURI("/a/b/dir1/file.txt") 

116 self.assertIsNone(not_child.relative_to(parent)) 

117 self.assertFalse(not_child.isdir()) 

118 

119 not_directory = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

120 self.assertIsNone(child.relative_to(not_directory)) 

121 

122 # Relative URIs 

123 parent = ButlerURI("a/b/", forceAbsolute=False) 

124 child = ButlerURI("a/b/c/d.txt", forceAbsolute=False) 

125 self.assertFalse(child.scheme) 

126 self.assertEqual(child.relative_to(parent), "c/d.txt") 

127 

128 # File URI and schemeless URI 

129 parent = ButlerURI("file:/a/b/c/") 

130 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

131 

132 # If the child is relative and the parent is absolute we assume 

133 # that the child is a child of the parent unless it uses ".." 

134 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

135 

136 child = ButlerURI("../e/f/g.txt", forceAbsolute=False) 

137 self.assertIsNone(child.relative_to(parent)) 

138 

139 child = ButlerURI("../c/e/f/g.txt", forceAbsolute=False) 

140 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

141 

142 # Test non-file root with relative path. 

143 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

144 parent = ButlerURI("s3://hello/a/b/c/") 

145 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

146 

147 # Test with different netloc 

148 child = ButlerURI("http://my.host/a/b/c.txt") 

149 parent = ButlerURI("http://other.host/a/") 

150 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

151 

152 # Schemeless absolute child. 

153 # Schemeless absolute URI is constructed using root= parameter. 

154 parent = ButlerURI("file:///a/b/c/") 

155 child = ButlerURI("d/e.txt", root=parent) 

156 self.assertEqual( 

157 child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})" 

158 ) 

159 

160 parent = ButlerURI("c/", root="/a/b/") 

161 self.assertEqual( 

162 child.relative_to(parent), "d/e.txt", f"{child}.relative_to({parent})" 

163 ) 

164 

165 # Absolute schemeless child with relative parent will always fail. 

166 parent = ButlerURI("d/e.txt", forceAbsolute=False) 

167 self.assertIsNone(child.relative_to(parent), f"{child}.relative_to({parent})") 

168 

169 def testParents(self): 

170 """Test of splitting and parent walking.""" 

171 parent = ButlerURI(self.tmpdir, forceDirectory=True, forceAbsolute=True) 

172 child_file = parent.join("subdir/file.txt") 

173 self.assertFalse(child_file.isdir()) 

174 child_subdir, file = child_file.split() 

175 self.assertEqual(file, "file.txt") 

176 self.assertTrue(child_subdir.isdir()) 

177 self.assertEqual(child_file.dirname(), child_subdir) 

178 self.assertEqual(child_file.basename(), file) 

179 self.assertEqual(child_file.parent(), child_subdir) 

180 derived_parent = child_subdir.parent() 

181 self.assertEqual(derived_parent, parent) 

182 self.assertTrue(derived_parent.isdir()) 

183 self.assertEqual(child_file.parent().parent(), parent) 

184 

185 def testEnvVar(self): 

186 """Test that environment variables are expanded.""" 

187 

188 with unittest.mock.patch.dict(os.environ, {"MY_TEST_DIR": "/a/b/c"}): 

189 uri = ButlerURI("${MY_TEST_DIR}/d.txt") 

190 self.assertEqual(uri.path, "/a/b/c/d.txt") 

191 self.assertEqual(uri.scheme, "file") 

192 

193 # This will not expand 

194 uri = ButlerURI("${MY_TEST_DIR}/d.txt", forceAbsolute=False) 

195 self.assertEqual(uri.path, "${MY_TEST_DIR}/d.txt") 

196 self.assertFalse(uri.scheme) 

197 

198 def testMkdir(self): 

199 tmpdir = ButlerURI(self.tmpdir) 

200 newdir = tmpdir.join("newdir/seconddir") 

201 newdir.mkdir() 

202 self.assertTrue(newdir.exists()) 

203 newfile = newdir.join("temp.txt") 

204 newfile.write("Data".encode()) 

205 self.assertTrue(newfile.exists()) 

206 

207 def testTransfer(self): 

208 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

209 content = "Content is some content\nwith something to say\n\n" 

210 src.write(content.encode()) 

211 

212 for mode in ("copy", "link", "hardlink", "symlink", "relsymlink"): 

213 dest = ButlerURI(os.path.join(self.tmpdir, f"dest_{mode}.txt")) 

214 dest.transfer_from(src, transfer=mode) 

215 self.assertTrue( 

216 dest.exists(), f"Check that {dest} exists (transfer={mode})" 

217 ) 

218 

219 with open(dest.ospath, "r") as fh: 

220 new_content = fh.read() 

221 self.assertEqual(new_content, content) 

222 

223 if mode in ("symlink", "relsymlink"): 

224 self.assertTrue( 

225 os.path.islink(dest.ospath), f"Check that {dest} is symlink" 

226 ) 

227 

228 # If the source and destination are hardlinks of each other 

229 # the transfer should work even if overwrite=False. 

230 if mode in ("link", "hardlink"): 

231 dest.transfer_from(src, transfer=mode) 

232 else: 

233 with self.assertRaises( 

234 FileExistsError, 

235 msg=f"Overwrite of {dest} should not be allowed ({mode})", 

236 ): 

237 dest.transfer_from(src, transfer=mode) 

238 

239 dest.transfer_from(src, transfer=mode, overwrite=True) 

240 

241 os.remove(dest.ospath) 

242 

243 b = src.read() 

244 self.assertEqual(b.decode(), new_content) 

245 

246 nbytes = 10 

247 subset = src.read(size=nbytes) 

248 self.assertEqual(len(subset), nbytes) 

249 self.assertEqual(subset.decode(), content[:nbytes]) 

250 

251 with self.assertRaises(ValueError): 

252 src.transfer_from(src, transfer="unknown") 

253 

254 def testTransferIdentical(self): 

255 """Test overwrite of identical files.""" 

256 dir1 = ButlerURI(os.path.join(self.tmpdir, "dir1"), forceDirectory=True) 

257 dir1.mkdir() 

258 dir2 = os.path.join(self.tmpdir, "dir2") 

259 os.symlink(dir1.ospath, dir2) 

260 

261 # Write a test file. 

262 src_file = dir1.join("test.txt") 

263 content = "0123456" 

264 src_file.write(content.encode()) 

265 

266 # Construct URI to destination that should be identical. 

267 dest_file = ButlerURI(os.path.join(dir2), forceDirectory=True).join("test.txt") 

268 self.assertTrue(dest_file.exists()) 

269 self.assertNotEqual(src_file, dest_file) 

270 

271 # Transfer it over itself. 

272 dest_file.transfer_from(src_file, transfer="symlink", overwrite=True) 

273 new_content = dest_file.read().decode() 

274 self.assertEqual(content, new_content) 

275 

276 def testResource(self): 

277 u = ButlerURI("resource://lsst.daf.butler/configs/datastore.yaml") 

278 self.assertTrue(u.exists(), f"Check {u} exists") 

279 

280 content = u.read().decode() 

281 self.assertTrue(content.startswith("datastore:")) 

282 

283 truncated = u.read(size=9).decode() 

284 self.assertEqual(truncated, "datastore") 

285 

286 d = ButlerURI("resource://lsst.daf.butler/configs", forceDirectory=True) 

287 self.assertTrue(u.exists(), f"Check directory {d} exists") 

288 

289 j = d.join("datastore.yaml") 

290 self.assertEqual(u, j) 

291 self.assertFalse(j.dirLike) 

292 self.assertFalse(j.isdir()) 

293 not_there = d.join("not-there.yaml") 

294 self.assertFalse(not_there.exists()) 

295 

296 bad = ButlerURI("resource://bad.module/not.yaml") 

297 multi = ButlerURI.mexists([u, bad, not_there]) 

298 self.assertTrue(multi[u]) 

299 self.assertFalse(multi[bad]) 

300 self.assertFalse(multi[not_there]) 

301 

302 def testEscapes(self): 

303 """Special characters in file paths""" 

304 src = ButlerURI("bbb/???/test.txt", root=self.tmpdir, forceAbsolute=True) 

305 self.assertFalse(src.scheme) 

306 src.write(b"Some content") 

307 self.assertTrue(src.exists()) 

308 

309 # abspath always returns a file scheme 

310 file = src.abspath() 

311 self.assertTrue(file.exists()) 

312 self.assertIn("???", file.ospath) 

313 self.assertNotIn("???", file.path) 

314 

315 file = file.updatedFile("tests??.txt") 

316 self.assertNotIn("??.txt", file.path) 

317 file.write(b"Other content") 

318 self.assertEqual(file.read(), b"Other content") 

319 

320 src = src.updatedFile("tests??.txt") 

321 self.assertIn("??.txt", src.path) 

322 self.assertEqual( 

323 file.read(), src.read(), f"reading from {file.ospath} and {src.ospath}" 

324 ) 

325 

326 # File URI and schemeless URI 

327 parent = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/")) 

328 child = ButlerURI("e/f/g.txt", forceAbsolute=False) 

329 self.assertEqual(child.relative_to(parent), "e/f/g.txt") 

330 

331 child = ButlerURI("e/f??#/g.txt", forceAbsolute=False) 

332 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

333 

334 child = ButlerURI("file:" + urllib.parse.quote("/a/b/c/de/??/e/f??#/g.txt")) 

335 self.assertEqual(child.relative_to(parent), "e/f??#/g.txt") 

336 

337 self.assertEqual(child.relativeToPathRoot, "a/b/c/de/??/e/f??#/g.txt") 

338 

339 # Schemeless so should not quote 

340 dir = ButlerURI( 

341 "bbb/???/", root=self.tmpdir, forceAbsolute=True, forceDirectory=True 

342 ) 

343 self.assertIn("???", dir.ospath) 

344 self.assertIn("???", dir.path) 

345 self.assertFalse(dir.scheme) 

346 

347 # dir.join() morphs into a file scheme 

348 new = dir.join("test_j.txt") 

349 self.assertIn("???", new.ospath, f"Checking {new}") 

350 new.write(b"Content") 

351 

352 new2name = "###/test??.txt" 

353 new2 = dir.join(new2name) 

354 self.assertIn("???", new2.ospath) 

355 new2.write(b"Content") 

356 self.assertTrue(new2.ospath.endswith(new2name)) 

357 self.assertEqual(new.read(), new2.read()) 

358 

359 fdir = dir.abspath() 

360 self.assertNotIn("???", fdir.path) 

361 self.assertIn("???", fdir.ospath) 

362 self.assertEqual(fdir.scheme, "file") 

363 fnew = dir.join("test_jf.txt") 

364 fnew.write(b"Content") 

365 

366 fnew2 = fdir.join(new2name) 

367 fnew2.write(b"Content") 

368 self.assertTrue(fnew2.ospath.endswith(new2name)) 

369 self.assertNotIn("###", fnew2.path) 

370 

371 self.assertEqual(fnew.read(), fnew2.read()) 

372 

373 # Test that children relative to schemeless and file schemes 

374 # still return the same unquoted name 

375 self.assertEqual( 

376 fnew2.relative_to(fdir), new2name, f"{fnew2}.relative_to({fdir})" 

377 ) 

378 self.assertEqual( 

379 fnew2.relative_to(dir), new2name, f"{fnew2}.relative_to({dir})" 

380 ) 

381 self.assertEqual( 

382 new2.relative_to(fdir), new2name, f"{new2}.relative_to({fdir})" 

383 ) 

384 self.assertEqual(new2.relative_to(dir), new2name, f"{new2}.relative_to({dir})") 

385 

386 # Check for double quoting 

387 plus_path = "/a/b/c+d/" 

388 with self.assertLogs(level="WARNING"): 

389 uri = ButlerURI(urllib.parse.quote(plus_path), forceDirectory=True) 

390 self.assertEqual(uri.ospath, plus_path) 

391 

392 # Check that # is not escaped for schemeless URIs 

393 hash_path = "/a/b#/c&d#xyz" 

394 hpos = hash_path.rfind("#") 

395 uri = ButlerURI(hash_path) 

396 self.assertEqual(uri.ospath, hash_path[:hpos]) 

397 self.assertEqual(uri.fragment, hash_path[hpos + 1:]) 

398 

399 def testHash(self): 

400 """Test that we can store URIs in sets and as keys.""" 

401 uri1 = ButlerURI(TESTDIR) 

402 uri2 = uri1.join("test/") 

403 s = {uri1, uri2} 

404 self.assertIn(uri1, s) 

405 

406 d = {uri1: "1", uri2: "2"} 

407 self.assertEqual(d[uri2], "2") 

408 

409 def testWalk(self): 

410 """Test ButlerURI.walk().""" 

411 test_dir_uri = ButlerURI(TESTDIR) 

412 

413 file = test_dir_uri.join("config/basic/butler.yaml") 

414 found = list(ButlerURI.findFileResources([file])) 

415 self.assertEqual(found[0], file) 

416 

417 # Compare against the full local paths 

418 expected = set( 

419 p 

420 for p in glob.glob(os.path.join(TESTDIR, "config", "**"), recursive=True) 

421 if os.path.isfile(p) 

422 ) 

423 found = set( 

424 u.ospath for u in ButlerURI.findFileResources([test_dir_uri.join("config")]) 

425 ) 

426 self.assertEqual(found, expected) 

427 

428 # Now solely the YAML files 

429 expected_yaml = set( 

430 glob.glob(os.path.join(TESTDIR, "config", "**", "*.yaml"), recursive=True) 

431 ) 

432 found = set( 

433 u.ospath 

434 for u in ButlerURI.findFileResources( 

435 [test_dir_uri.join("config")], file_filter=r".*\.yaml$" 

436 ) 

437 ) 

438 self.assertEqual(found, expected_yaml) 

439 

440 # Now two explicit directories and a file 

441 expected = set( 

442 glob.glob( 

443 os.path.join(TESTDIR, "config", "**", "basic", "*.yaml"), recursive=True 

444 ) 

445 ) 

446 expected.update( 

447 set( 

448 glob.glob( 

449 os.path.join(TESTDIR, "config", "**", "templates", "*.yaml"), 

450 recursive=True, 

451 ) 

452 ) 

453 ) 

454 expected.add(file.ospath) 

455 

456 found = set( 

457 u.ospath 

458 for u in ButlerURI.findFileResources( 

459 [ 

460 file, 

461 test_dir_uri.join("config/basic"), 

462 test_dir_uri.join("config/templates"), 

463 ], 

464 file_filter=r".*\.yaml$", 

465 ) 

466 ) 

467 self.assertEqual(found, expected) 

468 

469 # Group by directory -- find everything and compare it with what 

470 # we expected to be there in total. We expect to find 9 directories 

471 # containing yaml files so make sure we only iterate 9 times. 

472 found_yaml = set() 

473 counter = 0 

474 for uris in ButlerURI.findFileResources( 

475 [file, test_dir_uri.join("config/")], file_filter=r".*\.yaml$", grouped=True 

476 ): 

477 found = set(u.ospath for u in uris) 

478 if found: 

479 counter += 1 

480 

481 found_yaml.update(found) 

482 

483 self.assertEqual(found_yaml, expected_yaml) 

484 self.assertEqual(counter, 9) 

485 

486 # Grouping but check that single files are returned in a single group 

487 # at the end 

488 file2 = test_dir_uri.join("config/templates/templates-bad.yaml") 

489 found = list( 

490 ButlerURI.findFileResources( 

491 [file, file2, test_dir_uri.join("config/dbAuth")], grouped=True 

492 ) 

493 ) 

494 self.assertEqual(len(found), 2) 

495 self.assertEqual(list(found[1]), [file, file2]) 

496 

497 with self.assertRaises(ValueError): 

498 list(file.walk()) 

499 

500 def testRootURI(self): 

501 """Test ButlerURI.root_uri().""" 

502 uri = ButlerURI("https://www.notexist.com:8080/file/test") 

503 uri2 = ButlerURI("s3://www.notexist.com/file/test") 

504 self.assertEqual(uri.root_uri().geturl(), "https://www.notexist.com:8080/") 

505 self.assertEqual(uri2.root_uri().geturl(), "s3://www.notexist.com/") 

506 

507 def testJoin(self): 

508 """Test .join method.""" 

509 

510 root_str = "s3://bucket/hsc/payload/" 

511 root = ButlerURI(root_str) 

512 

513 self.assertEqual(root.join("b/test.txt").geturl(), f"{root_str}b/test.txt") 

514 add_dir = root.join("b/c/d/") 

515 self.assertTrue(add_dir.isdir()) 

516 self.assertEqual(add_dir.geturl(), f"{root_str}b/c/d/") 

517 

518 quote_example = "b&c.t@x#t" 

519 needs_quote = root.join(quote_example) 

520 self.assertEqual(needs_quote.unquoted_path, f"/hsc/payload/{quote_example}") 

521 

522 other = ButlerURI("file://localhost/test.txt") 

523 self.assertEqual(root.join(other), other) 

524 self.assertEqual(other.join("b/new.txt").geturl(), "file://localhost/b/new.txt") 

525 

526 joined = ButlerURI("s3://bucket/hsc/payload/").join( 

527 ButlerURI("test.qgraph", forceAbsolute=False) 

528 ) 

529 self.assertEqual(joined, ButlerURI("s3://bucket/hsc/payload/test.qgraph")) 

530 

531 with self.assertRaises(ValueError): 

532 ButlerURI("s3://bucket/hsc/payload/").join(ButlerURI("test.qgraph")) 

533 

534 def testTemporary(self): 

535 with ButlerURI.temporary_uri(suffix=".json") as tmp: 

536 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

537 self.assertTrue(tmp.isabs(), f"uri: {tmp}") 

538 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

539 tmp.write(b"abcd") 

540 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

541 self.assertTrue(tmp.isTemporary) 

542 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

543 

544 tmpdir = ButlerURI(self.tmpdir, forceDirectory=True) 

545 with ButlerURI.temporary_uri(prefix=tmpdir, suffix=".yaml") as tmp: 

546 # Use a specified tmpdir and check it is okay for the file 

547 # to not be created. 

548 self.assertFalse(tmp.exists(), f"uri: {tmp}") 

549 self.assertTrue(tmpdir.exists(), f"uri: {tmpdir} still exists") 

550 

551 

552@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

553@mock_s3 

554class S3URITestCase(unittest.TestCase): 

555 """Tests involving S3""" 

556 

557 bucketName = "any_bucket" 

558 """Bucket name to use in tests""" 

559 

560 def setUp(self): 

561 # Local test directory 

562 self.tmpdir = makeTestTempDir(TESTDIR) 

563 

564 # set up some fake credentials if they do not exist 

565 self.usingDummyCredentials = setAwsEnvCredentials() 

566 

567 # MOTO needs to know that we expect Bucket bucketname to exist 

568 s3 = boto3.resource("s3") 

569 s3.create_bucket(Bucket=self.bucketName) 

570 

571 def tearDown(self): 

572 s3 = boto3.resource("s3") 

573 bucket = s3.Bucket(self.bucketName) 

574 try: 

575 bucket.objects.all().delete() 

576 except botocore.exceptions.ClientError as e: 

577 if e.response["Error"]["Code"] == "404": 

578 # the key was not reachable - pass 

579 pass 

580 else: 

581 raise 

582 

583 bucket = s3.Bucket(self.bucketName) 

584 bucket.delete() 

585 

586 # unset any potentially set dummy credentials 

587 if self.usingDummyCredentials: 

588 unsetAwsEnvCredentials() 

589 

590 shutil.rmtree(self.tmpdir, ignore_errors=True) 

591 

592 def makeS3Uri(self, path): 

593 return f"s3://{self.bucketName}/{path}" 

594 

595 def testTransfer(self): 

596 src = ButlerURI(os.path.join(self.tmpdir, "test.txt")) 

597 content = "Content is some content\nwith something to say\n\n" 

598 src.write(content.encode()) 

599 self.assertTrue(src.exists()) 

600 self.assertEqual(src.size(), len(content.encode())) 

601 

602 dest = ButlerURI(self.makeS3Uri("test.txt")) 

603 self.assertFalse(dest.exists()) 

604 

605 with self.assertRaises(FileNotFoundError): 

606 dest.size() 

607 

608 dest.transfer_from(src, transfer="copy") 

609 self.assertTrue(dest.exists()) 

610 

611 dest2 = ButlerURI(self.makeS3Uri("copied.txt")) 

612 dest2.transfer_from(dest, transfer="copy") 

613 self.assertTrue(dest2.exists()) 

614 

615 local = ButlerURI(os.path.join(self.tmpdir, "copied.txt")) 

616 local.transfer_from(dest2, transfer="copy") 

617 with open(local.ospath, "r") as fd: 

618 new_content = fd.read() 

619 self.assertEqual(new_content, content) 

620 

621 with self.assertRaises(ValueError): 

622 dest2.transfer_from(local, transfer="symlink") 

623 

624 b = dest.read() 

625 self.assertEqual(b.decode(), new_content) 

626 

627 nbytes = 10 

628 subset = dest.read(size=nbytes) 

629 self.assertEqual(len(subset), nbytes) # Extra byte comes back 

630 self.assertEqual(subset.decode(), content[:nbytes]) 

631 

632 with self.assertRaises(FileExistsError): 

633 dest.transfer_from(src, transfer="copy") 

634 

635 dest.transfer_from(src, transfer="copy", overwrite=True) 

636 

637 def testWalk(self): 

638 """Test that we can list an S3 bucket""" 

639 # Files we want to create 

640 expected = ("a/x.txt", "a/y.txt", "a/z.json", "a/b/w.txt", "a/b/c/d/v.json") 

641 expected_uris = [ButlerURI(self.makeS3Uri(path)) for path in expected] 

642 for uri in expected_uris: 

643 # Doesn't matter what we write 

644 uri.write("123".encode()) 

645 

646 # Find all the files in the a/ tree 

647 found = set( 

648 uri.path 

649 for uri in ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("a/"))]) 

650 ) 

651 self.assertEqual(found, {uri.path for uri in expected_uris}) 

652 

653 # Find all the files in the a/ tree but group by folder 

654 found = ButlerURI.findFileResources( 

655 [ButlerURI(self.makeS3Uri("a/"))], grouped=True 

656 ) 

657 expected = ( 

658 ("/a/x.txt", "/a/y.txt", "/a/z.json"), 

659 ("/a/b/w.txt",), 

660 ("/a/b/c/d/v.json",), 

661 ) 

662 

663 for got, expect in zip(found, expected): 

664 self.assertEqual(tuple(u.path for u in got), expect) 

665 

666 # Find only JSON files 

667 found = set( 

668 uri.path 

669 for uri in ButlerURI.findFileResources( 

670 [ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$" 

671 ) 

672 ) 

673 self.assertEqual( 

674 found, {uri.path for uri in expected_uris if uri.path.endswith(".json")} 

675 ) 

676 

677 # JSON files grouped by directory 

678 found = ButlerURI.findFileResources( 

679 [ButlerURI(self.makeS3Uri("a/"))], file_filter=r"\.json$", grouped=True 

680 ) 

681 expected = (("/a/z.json",), ("/a/b/c/d/v.json",)) 

682 

683 for got, expect in zip(found, expected): 

684 self.assertEqual(tuple(u.path for u in got), expect) 

685 

686 # Check pagination works with large numbers of files. S3 API limits 

687 # us to 1000 response per list_objects call so create lots of files 

688 created = set() 

689 counter = 1 

690 n_dir1 = 1100 

691 while counter <= n_dir1: 

692 new = ButlerURI(self.makeS3Uri(f"test/file{counter:04d}.txt")) 

693 new.write(f"{counter}".encode()) 

694 created.add(str(new)) 

695 counter += 1 

696 counter = 1 

697 # Put some in a subdirectory to make sure we are looking in a 

698 # hierarchy. 

699 n_dir2 = 100 

700 while counter <= n_dir2: 

701 new = ButlerURI(self.makeS3Uri(f"test/subdir/file{counter:04d}.txt")) 

702 new.write(f"{counter}".encode()) 

703 created.add(str(new)) 

704 counter += 1 

705 

706 found = ButlerURI.findFileResources([ButlerURI(self.makeS3Uri("test/"))]) 

707 self.assertEqual({str(u) for u in found}, created) 

708 

709 # Again with grouping. 

710 found = list( 

711 ButlerURI.findFileResources( 

712 [ButlerURI(self.makeS3Uri("test/"))], grouped=True 

713 ) 

714 ) 

715 self.assertEqual(len(found), 2) 

716 dir_1 = list(found[0]) 

717 dir_2 = list(found[1]) 

718 self.assertEqual(len(dir_1), n_dir1) 

719 self.assertEqual(len(dir_2), n_dir2) 

720 

721 def testWrite(self): 

722 s3write = ButlerURI(self.makeS3Uri("created.txt")) 

723 content = "abcdefghijklmnopqrstuv\n" 

724 s3write.write(content.encode()) 

725 self.assertEqual(s3write.read().decode(), content) 

726 

727 def testTemporary(self): 

728 s3root = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

729 with ButlerURI.temporary_uri(prefix=s3root, suffix=".json") as tmp: 

730 self.assertEqual(tmp.getExtension(), ".json", f"uri: {tmp}") 

731 self.assertEqual(tmp.scheme, "s3", f"uri: {tmp}") 

732 self.assertEqual(tmp.parent(), s3root) 

733 basename = tmp.basename() 

734 content = "abcd" 

735 tmp.write(content.encode()) 

736 self.assertTrue(tmp.exists(), f"uri: {tmp}") 

737 self.assertFalse(tmp.exists()) 

738 

739 # Again without writing anything, to check that there is no complaint 

740 # on exit of context manager. 

741 with ButlerURI.temporary_uri(prefix=s3root, suffix=".json") as tmp: 

742 self.assertFalse(tmp.exists()) 

743 # Check that the file has a different name than before. 

744 self.assertNotEqual(tmp.basename(), basename, f"uri: {tmp}") 

745 self.assertFalse(tmp.exists()) 

746 

747 def testRelative(self): 

748 """Check that we can get subpaths back from two URIs""" 

749 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

750 child = ButlerURI(self.makeS3Uri("rootdir/dir1/file.txt")) 

751 

752 self.assertEqual(child.relative_to(parent), "dir1/file.txt") 

753 

754 not_child = ButlerURI(self.makeS3Uri("/a/b/dir1/file.txt")) 

755 self.assertFalse(not_child.relative_to(parent)) 

756 

757 not_s3 = ButlerURI(os.path.join(self.tmpdir, "dir1", "file2.txt")) 

758 self.assertFalse(child.relative_to(not_s3)) 

759 

760 def testQuoting(self): 

761 """Check that quoting works.""" 

762 parent = ButlerURI(self.makeS3Uri("rootdir"), forceDirectory=True) 

763 subpath = "rootdir/dir1+/file?.txt" 

764 child = ButlerURI(self.makeS3Uri(urllib.parse.quote(subpath))) 

765 

766 self.assertEqual(child.relative_to(parent), "dir1+/file?.txt") 

767 self.assertEqual(child.basename(), "file?.txt") 

768 self.assertEqual(child.relativeToPathRoot, subpath) 

769 self.assertIn("%", child.path) 

770 self.assertEqual(child.unquoted_path, "/" + subpath) 

771 

772 

773# Mock required environment variables during tests 

774class WebdavURITestCase(unittest.TestCase): 

775 def setUp(self): 

776 serverRoot = "www.not-exists.orgx" 

777 existingFolderName = "existingFolder" 

778 existingFileName = "existingFile" 

779 notExistingFileName = "notExistingFile" 

780 

781 self.baseURL = ButlerURI(f"https://{serverRoot}", forceDirectory=True) 

782 self.existingFileButlerURI = ButlerURI( 

783 f"https://{serverRoot}/{existingFolderName}/{existingFileName}" 

784 ) 

785 self.notExistingFileButlerURI = ButlerURI( 

786 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}" 

787 ) 

788 self.existingFolderButlerURI = ButlerURI( 

789 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True 

790 ) 

791 self.notExistingFolderButlerURI = ButlerURI( 

792 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True 

793 ) 

794 

795 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR)) 

796 

797 # Need to declare the options 

798 responses.add( 

799 responses.OPTIONS, 

800 self.baseURL.geturl(), 

801 status=200, 

802 headers={"DAV": "1,2,3"}, 

803 ) 

804 

805 # Used by ButlerhttpUri.exists() 

806 responses.add( 

807 responses.HEAD, 

808 self.existingFileButlerURI.geturl(), 

809 status=200, 

810 headers={"Content-Length": "1024"}, 

811 ) 

812 responses.add( 

813 responses.HEAD, self.notExistingFileButlerURI.geturl(), status=404 

814 ) 

815 

816 # Used by ButlerhttpUri.read() 

817 responses.add( 

818 responses.GET, 

819 self.existingFileButlerURI.geturl(), 

820 status=200, 

821 body=str.encode("It works!"), 

822 ) 

823 responses.add(responses.GET, self.notExistingFileButlerURI.geturl(), status=404) 

824 

825 # Used by ButlerhttpUri.write() 

826 responses.add(responses.PUT, self.existingFileButlerURI.geturl(), status=201) 

827 

828 # Used by ButlerhttpUri.transfer_from() 

829 responses.add( 

830 responses.Response( 

831 url=self.existingFileButlerURI.geturl(), 

832 method="COPY", 

833 headers={"Destination": self.existingFileButlerURI.geturl()}, 

834 status=201, 

835 ) 

836 ) 

837 responses.add( 

838 responses.Response( 

839 url=self.existingFileButlerURI.geturl(), 

840 method="COPY", 

841 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

842 status=201, 

843 ) 

844 ) 

845 responses.add( 

846 responses.Response( 

847 url=self.existingFileButlerURI.geturl(), 

848 method="MOVE", 

849 headers={"Destination": self.notExistingFileButlerURI.geturl()}, 

850 status=201, 

851 ) 

852 ) 

853 

854 # Used by ButlerhttpUri.remove() 

855 responses.add(responses.DELETE, self.existingFileButlerURI.geturl(), status=200) 

856 responses.add( 

857 responses.DELETE, self.notExistingFileButlerURI.geturl(), status=404 

858 ) 

859 

860 # Used by ButlerhttpUri.mkdir() 

861 responses.add( 

862 responses.HEAD, 

863 self.existingFolderButlerURI.geturl(), 

864 status=200, 

865 headers={"Content-Length": "1024"}, 

866 ) 

867 responses.add( 

868 responses.HEAD, 

869 self.baseURL.geturl(), 

870 status=200, 

871 headers={"Content-Length": "1024"}, 

872 ) 

873 responses.add( 

874 responses.HEAD, self.notExistingFolderButlerURI.geturl(), status=404 

875 ) 

876 responses.add( 

877 responses.Response( 

878 url=self.notExistingFolderButlerURI.geturl(), method="MKCOL", status=201 

879 ) 

880 ) 

881 responses.add( 

882 responses.Response( 

883 url=self.existingFolderButlerURI.geturl(), method="MKCOL", status=403 

884 ) 

885 ) 

886 

887 # Used by ButlerHttpURI._do_put() 

888 self.redirectPathNoExpect = ButlerURI( 

889 f"https://{serverRoot}/redirect-no-expect/file" 

890 ) 

891 self.redirectPathExpect = ButlerURI( 

892 f"https://{serverRoot}/redirect-expect/file" 

893 ) 

894 redirected_url = f"https://{serverRoot}/redirect/location" 

895 responses.add( 

896 responses.PUT, 

897 self.redirectPathNoExpect.geturl(), 

898 headers={"Location": redirected_url}, 

899 status=307, 

900 ) 

901 responses.add( 

902 responses.PUT, 

903 self.redirectPathExpect.geturl(), 

904 headers={"Location": redirected_url}, 

905 status=307, 

906 match=[ 

907 responses.matchers.header_matcher( 

908 {"Content-Length": "0", "Expect": "100-continue"} 

909 ) 

910 ], 

911 ) 

912 responses.add(responses.PUT, redirected_url, status=202) 

913 

914 def tearDown(self): 

915 if self.tmpdir and self.tmpdir.isLocal: 

916 removeTestTempDir(self.tmpdir.ospath) 

917 

918 @responses.activate 

919 def testExists(self): 

920 

921 self.assertTrue(self.existingFileButlerURI.exists()) 

922 self.assertFalse(self.notExistingFileButlerURI.exists()) 

923 

924 self.assertEqual(self.existingFileButlerURI.size(), 1024) 

925 with self.assertRaises(FileNotFoundError): 

926 self.notExistingFileButlerURI.size() 

927 

928 @responses.activate 

929 def testRemove(self): 

930 

931 self.assertIsNone(self.existingFileButlerURI.remove()) 

932 with self.assertRaises(FileNotFoundError): 

933 self.notExistingFileButlerURI.remove() 

934 

935 @responses.activate 

936 def testMkdir(self): 

937 

938 # The mock means that we can't check this now exists 

939 self.notExistingFolderButlerURI.mkdir() 

940 

941 # This should do nothing 

942 self.existingFolderButlerURI.mkdir() 

943 

944 with self.assertRaises(ValueError): 

945 self.notExistingFileButlerURI.mkdir() 

946 

947 @responses.activate 

948 def testRead(self): 

949 

950 self.assertEqual(self.existingFileButlerURI.read().decode(), "It works!") 

951 self.assertNotEqual(self.existingFileButlerURI.read().decode(), "Nope.") 

952 with self.assertRaises(FileNotFoundError): 

953 self.notExistingFileButlerURI.read() 

954 

955 # Run this twice to ensure use of cache in code coverage 

956 for _ in (1, 2): 

957 with self.existingFileButlerURI.as_local() as local_uri: 

958 self.assertTrue(local_uri.isLocal) 

959 content = local_uri.read().decode() 

960 self.assertEqual(content, "It works!") 

961 

962 # Ensure the LSST_RESOURCES_TMPDIR environment variable is used if set 

963 # (requires to reset the cache in _TMPDIR) 

964 httpUri._TMPDIR = None 

965 tmpdir = makeTestTempDir(TESTDIR) 

966 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": tmpdir}): 

967 with self.existingFileButlerURI.as_local() as local_uri: 

968 self.assertTrue(local_uri.isLocal) 

969 content = local_uri.read().decode() 

970 self.assertEqual(content, "It works!") 

971 self.assertIsNotNone(local_uri.relative_to(ButlerURI(tmpdir))) 

972 

973 @responses.activate 

974 def testWrite(self): 

975 

976 self.assertIsNone( 

977 self.existingFileButlerURI.write(data=str.encode("Some content.")) 

978 ) 

979 with self.assertRaises(FileExistsError): 

980 self.existingFileButlerURI.write( 

981 data=str.encode("Some content."), overwrite=False 

982 ) 

983 

984 @responses.activate 

985 def test_do_put_with_redirection(self): 

986 

987 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER. 

988 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None) 

989 importlib.reload(httpUri) 

990 body = str.encode("any contents") 

991 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body)) 

992 

993 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER. 

994 with unittest.mock.patch.dict( 

995 os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True 

996 ): 

997 importlib.reload(httpUri) 

998 self.assertIsNone(self.redirectPathExpect._do_put(data=body)) 

999 

1000 @responses.activate 

1001 def testTransfer(self): 

1002 

1003 self.assertIsNone( 

1004 self.notExistingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

1005 ) 

1006 self.assertIsNone( 

1007 self.notExistingFileButlerURI.transfer_from( 

1008 src=self.existingFileButlerURI, transfer="move" 

1009 ) 

1010 ) 

1011 with self.assertRaises(FileExistsError): 

1012 self.existingFileButlerURI.transfer_from(src=self.existingFileButlerURI) 

1013 with self.assertRaises(ValueError): 

1014 self.notExistingFileButlerURI.transfer_from( 

1015 src=self.existingFileButlerURI, transfer="unsupported" 

1016 ) 

1017 

1018 def testParent(self): 

1019 

1020 self.assertEqual( 

1021 self.existingFolderButlerURI.geturl(), 

1022 self.notExistingFileButlerURI.parent().geturl(), 

1023 ) 

1024 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl()) 

1025 self.assertEqual( 

1026 self.existingFileButlerURI.parent().geturl(), 

1027 self.existingFileButlerURI.dirname().geturl(), 

1028 ) 

1029 

1030 def test_send_expect_header(self): 

1031 

1032 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from 

1033 # the environment. 

1034 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None) 

1035 importlib.reload(httpUri) 

1036 self.assertFalse(httpUri._SEND_EXPECT_HEADER_ON_PUT) 

1037 

1038 with unittest.mock.patch.dict( 

1039 os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True 

1040 ): 

1041 importlib.reload(httpUri) 

1042 self.assertTrue(httpUri._SEND_EXPECT_HEADER_ON_PUT) 

1043 

1044 def test_timeout(self): 

1045 

1046 connect_timeout = 100 

1047 read_timeout = 200 

1048 with unittest.mock.patch.dict( 

1049 os.environ, 

1050 { 

1051 "LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), 

1052 "LSST_HTTP_TIMEOUT_READ": str(read_timeout), 

1053 }, 

1054 clear=True, 

1055 ): 

1056 # Force module reload to initialize TIMEOUT. 

1057 importlib.reload(httpUri) 

1058 self.assertEqual( 

1059 httpUri.TIMEOUT, 

1060 (connect_timeout, read_timeout), 

1061 ) 

1062 

1063 def test_is_protected(self): 

1064 

1065 _is_protected = httpUri._is_protected 

1066 self.assertFalse(_is_protected("/this-file-does-not-exist")) 

1067 

1068 with tempfile.NamedTemporaryFile( 

1069 mode="wt", dir=self.tmpdir.ospath, delete=False 

1070 ) as f: 

1071 f.write("XXXX") 

1072 file_path = f.name 

1073 

1074 os.chmod(file_path, stat.S_IRUSR) 

1075 self.assertTrue(_is_protected(file_path)) 

1076 

1077 for mode in ( 

1078 stat.S_IRGRP, 

1079 stat.S_IWGRP, 

1080 stat.S_IXGRP, 

1081 stat.S_IROTH, 

1082 stat.S_IWOTH, 

1083 stat.S_IXOTH, 

1084 ): 

1085 os.chmod(file_path, stat.S_IRUSR | mode) 

1086 self.assertFalse(_is_protected(file_path)) 

1087 

1088 

1089class WebdavUtilsTestCase(unittest.TestCase): 

1090 """Test for the Webdav related utilities.""" 

1091 

1092 serverRoot = "www.lsstwithwebdav.orgx" 

1093 wrongRoot = "www.lsstwithoutwebdav.org" 

1094 

1095 def setUp(self): 

1096 responses.add( 

1097 responses.OPTIONS, 

1098 f"https://{self.serverRoot}", 

1099 status=200, 

1100 headers={"DAV": "1,2,3"}, 

1101 ) 

1102 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200) 

1103 

1104 @responses.activate 

1105 def test_is_webdav_endpoint(self): 

1106 

1107 _is_webdav_endpoint = httpUri._is_webdav_endpoint 

1108 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}")) 

1109 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}")) 

1110 

1111 

1112class BearerTokenAuthTestCase(unittest.TestCase): 

1113 """Test for the BearerTokenAuth class.""" 

1114 

1115 def setUp(self): 

1116 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR)) 

1117 self.token = "ABCDE1234" 

1118 

1119 def tearDown(self): 

1120 if self.tmpdir and self.tmpdir.isLocal: 

1121 removeTestTempDir(self.tmpdir.ospath) 

1122 

1123 def test_empty_token(self): 

1124 """Ensure that when no token is provided the request is not 

1125 modified. 

1126 """ 

1127 auth = httpUri.BearerTokenAuth(None) 

1128 auth._refresh() 

1129 self.assertIsNone(auth._token) 

1130 self.assertIsNone(auth._path) 

1131 req = requests.Request("GET", "https://example.org") 

1132 self.assertEqual(auth(req), req) 

1133 

1134 def test_token_value(self): 

1135 """Ensure that when a token value is provided, the 'Authorization' 

1136 header is added to the requests. 

1137 """ 

1138 auth = httpUri.BearerTokenAuth(self.token) 

1139 req = auth(requests.Request("GET", "https://example.org").prepare()) 

1140 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

1141 

1142 def test_token_file(self): 

1143 """Ensure when the provided token is a file path, its contents is 

1144 correctly used in the the 'Authorization' header of the requests. 

1145 """ 

1146 with tempfile.NamedTemporaryFile( 

1147 mode="wt", dir=self.tmpdir.ospath, delete=False 

1148 ) as f: 

1149 f.write(self.token) 

1150 token_file_path = f.name 

1151 

1152 # Ensure the request's "Authorization" header is set with the right 

1153 # token value 

1154 os.chmod(token_file_path, stat.S_IRUSR) 

1155 auth = httpUri.BearerTokenAuth(token_file_path) 

1156 req = auth(requests.Request("GET", "https://example.org").prepare()) 

1157 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

1158 

1159 # Ensure an exception is raised if either group or other can read the 

1160 # token file 

1161 for mode in ( 

1162 stat.S_IRGRP, 

1163 stat.S_IWGRP, 

1164 stat.S_IXGRP, 

1165 stat.S_IROTH, 

1166 stat.S_IWOTH, 

1167 stat.S_IXOTH, 

1168 ): 

1169 os.chmod(token_file_path, stat.S_IRUSR | mode) 

1170 with self.assertRaises(PermissionError): 

1171 httpUri.BearerTokenAuth(token_file_path) 

1172 

1173 

1174class SessionStoreTestCase(unittest.TestCase): 

1175 """Test for the SessionStore class.""" 

1176 

1177 def setUp(self): 

1178 self.tmpdir = ButlerURI(makeTestTempDir(TESTDIR)) 

1179 self.rpath = ButlerURI("https://example.org") 

1180 

1181 def tearDown(self): 

1182 if self.tmpdir and self.tmpdir.isLocal: 

1183 removeTestTempDir(self.tmpdir.ospath) 

1184 

1185 def test_ca_cert_bundle(self): 

1186 """Ensure a certificate authorities bundle is used to authentify 

1187 the remote server. 

1188 """ 

1189 with tempfile.NamedTemporaryFile( 

1190 mode="wt", dir=self.tmpdir.ospath, delete=False 

1191 ) as f: 

1192 f.write("CERT BUNDLE") 

1193 cert_bundle = f.name 

1194 

1195 with unittest.mock.patch.dict( 

1196 os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True 

1197 ): 

1198 session = httpUri.SessionStore().get(self.rpath) 

1199 self.assertEqual(session.verify, cert_bundle) 

1200 

1201 def test_user_cert(self): 

1202 """Ensure if user certificate and private key are provided, they are 

1203 used for authenticating the client. 

1204 """ 

1205 

1206 # Create mock certificate and private key files. 

1207 with tempfile.NamedTemporaryFile( 

1208 mode="wt", dir=self.tmpdir.ospath, delete=False 

1209 ) as f: 

1210 f.write("CERT") 

1211 client_cert = f.name 

1212 

1213 with tempfile.NamedTemporaryFile( 

1214 mode="wt", dir=self.tmpdir.ospath, delete=False 

1215 ) as f: 

1216 f.write("KEY") 

1217 client_key = f.name 

1218 

1219 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY 

1220 # must be initialized. 

1221 with unittest.mock.patch.dict( 

1222 os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True 

1223 ): 

1224 with self.assertRaises(ValueError): 

1225 httpUri.SessionStore().get(self.rpath) 

1226 

1227 with unittest.mock.patch.dict( 

1228 os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True 

1229 ): 

1230 with self.assertRaises(ValueError): 

1231 httpUri.SessionStore().get(self.rpath) 

1232 

1233 # Check private key file must be accessible only by its owner. 

1234 with unittest.mock.patch.dict( 

1235 os.environ, 

1236 { 

1237 "LSST_HTTP_AUTH_CLIENT_CERT": client_cert, 

1238 "LSST_HTTP_AUTH_CLIENT_KEY": client_key, 

1239 }, 

1240 clear=True, 

1241 ): 

1242 # Ensure the session client certificate is initialized when 

1243 # only the owner can read the private key file. 

1244 os.chmod(client_key, stat.S_IRUSR) 

1245 session = httpUri.SessionStore().get(self.rpath) 

1246 self.assertEqual(session.cert[0], client_cert) 

1247 self.assertEqual(session.cert[1], client_key) 

1248 

1249 # Ensure an exception is raised if either group or other can access 

1250 # the private key file. 

1251 for mode in ( 

1252 stat.S_IRGRP, 

1253 stat.S_IWGRP, 

1254 stat.S_IXGRP, 

1255 stat.S_IROTH, 

1256 stat.S_IWOTH, 

1257 stat.S_IXOTH, 

1258 ): 

1259 os.chmod(client_key, stat.S_IRUSR | mode) 

1260 with self.assertRaises(PermissionError): 

1261 httpUri.SessionStore().get(self.rpath) 

1262 

1263 def test_token_env(self): 

1264 """Ensure when the token is provided via an environment variable 

1265 the sessions are equipped with a BearerTokenAuth. 

1266 """ 

1267 token = "ABCDE" 

1268 with unittest.mock.patch.dict( 

1269 os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True 

1270 ): 

1271 session = httpUri.SessionStore().get(self.rpath) 

1272 self.assertEqual(type(session.auth), httpUri.BearerTokenAuth) 

1273 self.assertEqual(session.auth._token, token) 

1274 self.assertIsNone(session.auth._path) 

1275 

1276 def test_sessions(self): 

1277 """Ensure the session caching mechanism works.""" 

1278 

1279 # Ensure the store provides a session for a given URL 

1280 root_url = "https://example.org" 

1281 store = httpUri.SessionStore() 

1282 session = store.get(ButlerURI(root_url)) 

1283 self.assertIsNotNone(session) 

1284 

1285 # Ensure the sessions retrieved from a single store with the same 

1286 # root URIs are equal 

1287 for u in (f"{root_url}", f"{root_url}/path/to/file"): 

1288 self.assertEqual(session, store.get(ButlerURI(u))) 

1289 

1290 # Ensure sessions retrieved for different root URIs are different 

1291 another_url = "https://another.example.org" 

1292 self.assertNotEqual(session, store.get(ButlerURI(another_url))) 

1293 

1294 # Ensure the sessions retrieved from a single store for URLs with 

1295 # different port numbers are different 

1296 root_url_with_port = f"{another_url}:12345" 

1297 session = store.get(ButlerURI(root_url_with_port)) 

1298 self.assertNotEqual(session, store.get(ButlerURI(another_url))) 

1299 

1300 # Ensure the sessions retrieved from a single store with the same 

1301 # root URIs (including port numbers) are equal 

1302 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"): 

1303 self.assertEqual(session, store.get(ButlerURI(u))) 

1304 

1305 

1306if __name__ == "__main__": 1306 ↛ 1307line 1306 didn't jump to line 1307, because the condition on line 1306 was never true

1307 unittest.main()