Coverage for tests/test_http.py: 15%

452 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-25 02:02 -0700

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import hashlib 

13import importlib 

14import io 

15import os.path 

16import random 

17import shutil 

18import socket 

19import stat 

20import string 

21import tempfile 

22import time 

23import unittest 

24import warnings 

25from threading import Thread 

26from typing import Callable, Tuple, cast 

27 

28try: 

29 from cheroot import wsgi 

30 from wsgidav.wsgidav_app import WsgiDAVApp 

31except ImportError: 

32 WsgiDAVApp = None 

33 

34import lsst.resources 

35import requests 

36import responses 

37from lsst.resources import ResourcePath 

38from lsst.resources._resourceHandles._httpResourceHandle import HttpReadResourceHandle 

39from lsst.resources.http import ( 

40 BearerTokenAuth, 

41 HttpResourcePathConfig, 

42 SessionStore, 

43 _is_protected, 

44 _is_webdav_endpoint, 

45) 

46from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

47from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

48 

49TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

50 

51 

52class GenericHttpTestCase(GenericTestCase, unittest.TestCase): 

53 scheme = "http" 

54 netloc = "server.example" 

55 

56 

57class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase): 

58 """Test with a real webDAV server, as opposed to mocking responses.""" 

59 

60 scheme = "http" 

61 

62 @classmethod 

63 def setUpClass(cls): 

64 cls.webdav_tmpdir = tempfile.mkdtemp(prefix="webdav-server-test-") 

65 cls.local_files_to_remove = [] 

66 cls.server_thread = None 

67 

68 # Disable warnings about socket connections left open. We purposedly 

69 # keep network connections to the remote server open and have no 

70 # means through the API exposed by Requests of actually close the 

71 # underlyng sockets to make tests pass without warning. 

72 warnings.filterwarnings(action="ignore", message=r"unclosed.*socket", category=ResourceWarning) 

73 

74 # Should we test against a running server? 

75 # 

76 # This is convenient for testing against real servers in the 

77 # developer environment by initializing the environment variable 

78 # LSST_RESOURCES_HTTP_TEST_SERVER_URL with the URL of the server, e.g. 

79 # https://dav.example.org:1234/path/to/top/dir 

80 if (test_endpoint := os.getenv("LSST_RESOURCES_HTTP_TEST_SERVER_URL")) is not None: 

81 # Run this test case against the specified server. 

82 uri = ResourcePath(test_endpoint) 

83 cls.scheme = uri.scheme 

84 cls.netloc = uri.netloc 

85 cls.base_path = uri.path 

86 elif WsgiDAVApp is not None: 

87 # WsgiDAVApp is available, launch a local server in its own 

88 # thread to expose a local temporary directory and run this 

89 # test case against it. 

90 cls.port_number = cls._get_port_number() 

91 cls.stop_webdav_server = False 

92 cls.server_thread = Thread( 

93 target=cls._serve_webdav, 

94 args=(cls, cls.webdav_tmpdir, cls.port_number, lambda: cls.stop_webdav_server), 

95 daemon=True, 

96 ) 

97 cls.server_thread.start() 

98 

99 # Wait for it to start 

100 time.sleep(1) 

101 

102 # Initialize the server endpoint 

103 cls.netloc = f"127.0.0.1:{cls.port_number}" 

104 else: 

105 cls.skipTest( 

106 cls, 

107 "neither WsgiDAVApp is available nor a webDAV test endpoint is configured to test against", 

108 ) 

109 

110 @classmethod 

111 def tearDownClass(cls): 

112 # Stop the WsgiDAVApp server, if any 

113 if WsgiDAVApp is not None: 

114 # Shut down of the webdav server and wait for the thread to exit 

115 cls.stop_webdav_server = True 

116 if cls.server_thread is not None: 

117 cls.server_thread.join() 

118 

119 # Remove local temporary files 

120 for file in cls.local_files_to_remove: 

121 if os.path.exists(file): 

122 os.remove(file) 

123 

124 # Remove temp dir 

125 if cls.webdav_tmpdir: 

126 shutil.rmtree(cls.webdav_tmpdir, ignore_errors=True) 

127 

128 # Reset the warnings filter. 

129 warnings.resetwarnings() 

130 

131 def tearDown(self): 

132 if self.tmpdir: 

133 self.tmpdir.remove() 

134 

135 # Clear sessions. Some sockets may be left open, because urllib3 

136 # doest not close in-flight connections. 

137 # See https://urllib3.readthedocs.io > API Reference > 

138 # Pool Manager > clear() 

139 # I cannot add the full URL here because it is longer than 79 

140 # characters. 

141 self.tmpdir._clear_sessions() 

142 

143 super().tearDown() 

144 

145 def test_dav_file_handle(self): 

146 # Upload a new file with known contents. 

147 contents = "These are some \n bytes to read" 

148 remote_file = self.tmpdir.join(self._get_file_name()) 

149 self.assertIsNone(remote_file.write(data=contents, overwrite=True)) 

150 

151 # Test that the correct handle is returned. 

152 with remote_file.open("rb") as handle: 

153 self.assertIsInstance(handle, HttpReadResourceHandle) 

154 

155 # Test reading byte ranges works 

156 with remote_file.open("rb") as handle: 

157 sub_contents = contents[:10] 

158 handle = cast(HttpReadResourceHandle, handle) 

159 result = handle.read(len(sub_contents)).decode() 

160 self.assertEqual(result, sub_contents) 

161 # Verify there is no internal buffer. 

162 self.assertIsNone(handle._completeBuffer) 

163 # Verify the position. 

164 self.assertEqual(handle.tell(), len(sub_contents)) 

165 

166 # Jump back to the beginning and test if reading the whole file 

167 # prompts the internal buffer to be read. 

168 handle.seek(0) 

169 self.assertEqual(handle.tell(), 0) 

170 result = handle.read().decode() 

171 self.assertIsNotNone(handle._completeBuffer) 

172 self.assertEqual(result, contents) 

173 

174 # Verify reading as a string handle works as expected. 

175 with remote_file.open("r") as handle: 

176 self.assertIsInstance(handle, io.TextIOWrapper) 

177 

178 handle = cast(io.TextIOWrapper, handle) 

179 self.assertIsInstance(handle.buffer, HttpReadResourceHandle) 

180 

181 # Check if string methods work. 

182 result = handle.read() 

183 self.assertEqual(result, contents) 

184 

185 # Verify that write modes invoke the default base method 

186 with remote_file.open("w") as handle: 

187 self.assertIsInstance(handle, io.StringIO) 

188 

189 def test_dav_is_dav_enpoint(self): 

190 # Ensure the server is a webDAV endpoint 

191 self.assertTrue(self.tmpdir.is_webdav_endpoint) 

192 

193 def test_dav_mkdir(self): 

194 # Check creation and deletion of an empty directory 

195 subdir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

196 self.assertIsNone(subdir.mkdir()) 

197 self.assertTrue(subdir.exists()) 

198 

199 # Creating an existing remote directory must succeed 

200 self.assertIsNone(subdir.mkdir()) 

201 

202 # Deletion of an existing directory must succeed 

203 self.assertIsNone(subdir.remove()) 

204 

205 # Deletion of an non-existing directory must succeed 

206 subir_not_exists = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

207 self.assertIsNone(subir_not_exists.remove()) 

208 

209 # Creation of a directory at a path where a file exists must raise 

210 file = self.tmpdir.join(self._get_file_name(), forceDirectory=False) 

211 file.write(data=None, overwrite=True) 

212 self.assertTrue(file.exists()) 

213 

214 existing_file = self.tmpdir.join(file.basename(), forceDirectory=True) 

215 with self.assertRaises(NotADirectoryError): 

216 self.assertIsNone(existing_file.mkdir()) 

217 

218 def test_dav_upload_download(self): 

219 # Test upload a randomly-generated file via write() with and without 

220 # overwrite 

221 local_file, file_size = self._generate_file() 

222 with open(local_file, "rb") as f: 

223 data = f.read() 

224 

225 remote_file = self.tmpdir.join(self._get_file_name()) 

226 self.assertIsNone(remote_file.write(data, overwrite=True)) 

227 self.assertTrue(remote_file.exists()) 

228 self.assertEqual(remote_file.size(), file_size) 

229 

230 # Write without overwrite must raise since target file exists 

231 with self.assertRaises(FileExistsError): 

232 remote_file.write(data, overwrite=False) 

233 

234 # Download the file we just uploaded. Compute and compare a digest of 

235 # the uploaded and downloaded data and ensure they match 

236 downloaded_data = remote_file.read() 

237 self.assertEqual(len(downloaded_data), file_size) 

238 upload_digest = self._compute_digest(data) 

239 download_digest = self._compute_digest(downloaded_data) 

240 self.assertEqual(upload_digest, download_digest) 

241 os.remove(local_file) 

242 

243 def test_dav_as_local(self): 

244 contents = str.encode("12345") 

245 remote_file = self.tmpdir.join(self._get_file_name()) 

246 self.assertIsNone(remote_file.write(data=contents, overwrite=True)) 

247 

248 local_path, is_temp = remote_file._as_local() 

249 self.assertTrue(is_temp) 

250 self.assertTrue(os.path.exists(local_path)) 

251 self.assertTrue(os.stat(local_path).st_size, len(contents)) 

252 self.assertEqual(ResourcePath(local_path).read(), contents) 

253 os.remove(local_path) 

254 

255 def test_dav_size(self): 

256 # Size of a non-existent file must raise. 

257 remote_file = self.tmpdir.join(self._get_file_name()) 

258 with self.assertRaises(FileNotFoundError): 

259 remote_file.size() 

260 

261 # Retrieving the size of a remote directory using a file-like path must 

262 # raise 

263 remote_dir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

264 self.assertIsNone(remote_dir.mkdir()) 

265 self.assertTrue(remote_dir.exists()) 

266 

267 dir_as_file = ResourcePath(remote_dir.geturl().rstrip("/"), forceDirectory=False) 

268 with self.assertRaises(IsADirectoryError): 

269 dir_as_file.size() 

270 

271 def test_dav_upload_creates_dir(self): 

272 # Uploading a file to a non existing directory must ensure its 

273 # parent directories are automatically created and upload succeeds 

274 non_existing_dir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

275 non_existing_dir = non_existing_dir.join(self._get_dir_name(), forceDirectory=True) 

276 non_existing_dir = non_existing_dir.join(self._get_dir_name(), forceDirectory=True) 

277 remote_file = non_existing_dir.join(self._get_file_name()) 

278 

279 local_file, file_size = self._generate_file() 

280 with open(local_file, "rb") as f: 

281 data = f.read() 

282 self.assertIsNone(remote_file.write(data, overwrite=True)) 

283 

284 self.assertTrue(remote_file.exists()) 

285 self.assertEqual(remote_file.size(), file_size) 

286 self.assertTrue(remote_file.parent().exists()) 

287 

288 downloaded_data = remote_file.read() 

289 upload_digest = self._compute_digest(data) 

290 download_digest = self._compute_digest(downloaded_data) 

291 self.assertEqual(upload_digest, download_digest) 

292 os.remove(local_file) 

293 

294 def test_dav_transfer_from(self): 

295 # Transfer from local file via "copy", with and without overwrite 

296 remote_file = self.tmpdir.join(self._get_file_name()) 

297 local_file, _ = self._generate_file() 

298 source_file = ResourcePath(local_file) 

299 self.assertIsNone(remote_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

300 self.assertTrue(remote_file.exists()) 

301 self.assertEqual(remote_file.size(), source_file.size()) 

302 with self.assertRaises(FileExistsError): 

303 remote_file.transfer_from(ResourcePath(local_file), transfer="copy", overwrite=False) 

304 

305 # Transfer from remote file via "copy", with and without overwrite 

306 source_file = remote_file 

307 target_file = self.tmpdir.join(self._get_file_name()) 

308 self.assertIsNone(target_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

309 self.assertTrue(target_file.exists()) 

310 self.assertEqual(target_file.size(), source_file.size()) 

311 

312 # Transfer without overwrite must raise since target resource exists 

313 with self.assertRaises(FileExistsError): 

314 target_file.transfer_from(source_file, transfer="copy", overwrite=False) 

315 

316 # Test transfer from local file via "move", with and without overwrite 

317 source_file = ResourcePath(local_file) 

318 source_size = source_file.size() 

319 target_file = self.tmpdir.join(self._get_file_name()) 

320 self.assertIsNone(target_file.transfer_from(source_file, transfer="move", overwrite=True)) 

321 self.assertTrue(target_file.exists()) 

322 self.assertEqual(target_file.size(), source_size) 

323 self.assertFalse(source_file.exists()) 

324 

325 # Test transfer without overwrite must raise since target resource 

326 # exists 

327 local_file, file_size = self._generate_file() 

328 with self.assertRaises(FileExistsError): 

329 source_file = ResourcePath(local_file) 

330 target_file.transfer_from(source_file, transfer="move", overwrite=False) 

331 

332 # Test transfer from remote file via "move" with and without overwrite 

333 # must succeed 

334 source_file = target_file 

335 source_size = source_file.size() 

336 target_file = self.tmpdir.join(self._get_file_name()) 

337 self.assertIsNone(target_file.transfer_from(source_file, transfer="move", overwrite=True)) 

338 self.assertTrue(target_file.exists()) 

339 self.assertEqual(target_file.size(), source_size) 

340 self.assertFalse(source_file.exists()) 

341 

342 # Transfer without overwrite must raise since target resource exists 

343 with self.assertRaises(FileExistsError): 

344 source_file = ResourcePath(local_file) 

345 target_file.transfer_from(source_file, transfer="move", overwrite=False) 

346 

347 def test_dav_handle(self): 

348 # Resource handle must succeed 

349 target_file = self.tmpdir.join(self._get_file_name()) 

350 data = "abcdefghi" 

351 self.assertIsNone(target_file.write(data, overwrite=True)) 

352 with target_file.open("rb") as handle: 

353 handle.seek(1) 

354 self.assertEqual(handle.read(4).decode("utf-8"), data[1:5]) 

355 

356 def test_dav_delete(self): 

357 # Deletion of an existing remote file must succeed 

358 local_file, file_size = self._generate_file() 

359 with open(local_file, "rb") as f: 

360 data = f.read() 

361 

362 remote_file = self.tmpdir.join(self._get_file_name()) 

363 self.assertIsNone(remote_file.write(data, overwrite=True)) 

364 self.assertTrue(remote_file.exists()) 

365 self.assertEqual(remote_file.size(), file_size) 

366 self.assertIsNone(remote_file.remove()) 

367 os.remove(local_file) 

368 

369 # Deletion of a non-existing remote file must succeed 

370 non_existing_file = self.tmpdir.join(self._get_file_name()) 

371 self.assertIsNone(non_existing_file.remove()) 

372 

373 # Deletion of a non-empty remote directory must succeed 

374 subdir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

375 self.assertIsNone(subdir.mkdir()) 

376 self.assertTrue(subdir.exists()) 

377 local_file, _ = self._generate_file() 

378 source_file = ResourcePath(local_file) 

379 target_file = self.tmpdir.join(self._get_file_name(), forceDirectory=True) 

380 self.assertIsNone(target_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

381 self.assertIsNone(subdir.remove()) 

382 self.assertFalse(subdir.exists()) 

383 os.remove(local_file) 

384 

385 @classmethod 

386 def _get_port_number(cls) -> int: 

387 """Return a port number the webDAV server can use to listen to.""" 

388 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 

389 s.bind(("127.0.0.1", 0)) 

390 s.listen() 

391 port = s.getsockname()[1] 

392 s.close() 

393 return port 

394 

395 def _serve_webdav(self, local_path: str, port: int, stop_webdav_server: Callable[[], bool]): 

396 """Start a local webDAV server, listening on http://localhost:port 

397 and exposing local_path. 

398 

399 This server only runs when this test class is instantiated, 

400 and then shuts down. The server must be started is a separate thread. 

401 

402 Parameters 

403 ---------- 

404 port : `int` 

405 The port number on which the server should listen 

406 local_path : `str` 

407 Path to an existing local directory for the server to expose. 

408 stop_webdav_server : `Callable[[], bool]` 

409 Boolean function which returns True when the server should be 

410 stopped. 

411 """ 

412 try: 

413 # Start the wsgi server in a separate thread 

414 config = { 

415 "host": "127.0.0.1", 

416 "port": port, 

417 "provider_mapping": {"/": local_path}, 

418 "http_authenticator": {"domain_controller": None}, 

419 "simple_dc": {"user_mapping": {"*": True}}, 

420 "verbose": 0, 

421 "lock_storage": False, 

422 "dir_browser": { 

423 "enable": False, 

424 "ms_sharepoint_support": False, 

425 "libre_office_support": False, 

426 "response_trailer": False, 

427 "davmount_links": False, 

428 }, 

429 } 

430 server = wsgi.Server(wsgi_app=WsgiDAVApp(config), bind_addr=(config["host"], config["port"])) 

431 t = Thread(target=server.start, daemon=True) 

432 t.start() 

433 

434 # Shut down the server when done: stop_webdav_server() returns 

435 # True when this test suite is being teared down 

436 while not stop_webdav_server(): 

437 time.sleep(1) 

438 except KeyboardInterrupt: 

439 # Caught Ctrl-C, shut down the server 

440 pass 

441 finally: 

442 server.stop() 

443 t.join() 

444 

445 @classmethod 

446 def _get_name(cls, prefix: str) -> str: 

447 alphabet = string.ascii_lowercase + string.digits 

448 return f"{prefix}-" + "".join(random.choices(alphabet, k=8)) 

449 

450 @classmethod 

451 def _get_dir_name(cls) -> str: 

452 """Return a randomly selected name for a file""" 

453 return cls._get_name(prefix="dir") 

454 

455 @classmethod 

456 def _get_file_name(cls) -> str: 

457 """Return a randomly selected name for a file""" 

458 return cls._get_name(prefix="file") 

459 

460 def _generate_file(self, remove_when_done=True) -> Tuple[str, int]: 

461 """Create a local file of random size with random contents. 

462 

463 Returns 

464 ------- 

465 path : `str` 

466 Path to local temporary file. The caller is responsible for 

467 removing the file when appropriate. 

468 size : `int` 

469 Size of the generated file, in bytes. 

470 """ 

471 megabyte = 1024 * 1024 

472 size = random.randint(2 * megabyte, 5 * megabyte) 

473 tmpfile, path = tempfile.mkstemp() 

474 self.assertEqual(os.write(tmpfile, os.urandom(size)), size) 

475 os.close(tmpfile) 

476 

477 if remove_when_done: 

478 self.local_files_to_remove.append(path) 

479 

480 return path, size 

481 

482 @classmethod 

483 def _compute_digest(cls, data: bytes) -> str: 

484 """Compute a SHA256 hash of data.""" 

485 m = hashlib.sha256() 

486 m.update(data) 

487 return m.hexdigest() 

488 

489 @classmethod 

490 def _is_server_running(cls, port: int) -> bool: 

491 """Return True if there is a server listening on local address 

492 127.0.0.1:<port>. 

493 """ 

494 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 

495 try: 

496 s.connect(("127.0.0.1", port)) 

497 return True 

498 except ConnectionRefusedError: 

499 return False 

500 

501 

502class HttpResourcePathConfigTestCase(unittest.TestCase): 

503 """Test for the HttpResourcePathConfig class.""" 

504 

505 def test_send_expect_header(self): 

506 # Ensure environment variable LSST_HTTP_PUT_SEND_EXPECT_HEADER is 

507 # inspected to initialize the HttpResourcePath config class. 

508 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

509 importlib.reload(lsst.resources.http) 

510 config = HttpResourcePathConfig() 

511 self.assertFalse(config.send_expect_on_put) 

512 

513 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True): 

514 importlib.reload(lsst.resources.http) 

515 config = HttpResourcePathConfig() 

516 self.assertTrue(config.send_expect_on_put) 

517 

518 def test_timeout(self): 

519 # Ensure that when the connect and read timeouts are not specified 

520 # the default values are stored in the config. 

521 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

522 importlib.reload(lsst.resources.http) 

523 config = HttpResourcePathConfig() 

524 self.assertEqual( 

525 config.timeout, 

526 (lsst.resources.http.DEFAULT_TIMEOUT_CONNECT, lsst.resources.http.DEFAULT_TIMEOUT_READ), 

527 ) 

528 

529 # Ensure that when both the connect and read timeouts are specified 

530 # they are stored in the config. 

531 connect_timeout, read_timeout = 100, 100 

532 with unittest.mock.patch.dict( 

533 os.environ, 

534 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)}, 

535 clear=True, 

536 ): 

537 # Force module reload. 

538 importlib.reload(lsst.resources.http) 

539 config = HttpResourcePathConfig() 

540 self.assertEqual(config.timeout, (connect_timeout, read_timeout)) 

541 

542 def test_front_end_connections(self): 

543 # Ensure that when the number of front end connections is not specified 

544 # the default is stored in the config. 

545 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

546 importlib.reload(lsst.resources.http) 

547 config = HttpResourcePathConfig() 

548 self.assertEqual( 

549 config.front_end_connections, int(lsst.resources.http.DEFAULT_FRONTEND_PERSISTENT_CONNECTIONS) 

550 ) 

551 

552 # Ensure that when the number of front end connections is specified 

553 # it is stored in the config. 

554 connections = 42 

555 with unittest.mock.patch.dict( 

556 os.environ, {"LSST_HTTP_FRONTEND_PERSISTENT_CONNECTIONS": str(connections)}, clear=True 

557 ): 

558 importlib.reload(lsst.resources.http) 

559 config = HttpResourcePathConfig() 

560 self.assertTrue(config.front_end_connections, connections) 

561 

562 def test_back_end_connections(self): 

563 # Ensure that when the number of back end connections is not specified 

564 # the default is stored in the config. 

565 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

566 importlib.reload(lsst.resources.http) 

567 config = HttpResourcePathConfig() 

568 self.assertEqual( 

569 config.back_end_connections, int(lsst.resources.http.DEFAULT_BACKEND_PERSISTENT_CONNECTIONS) 

570 ) 

571 

572 # Ensure that when the number of back end connections is specified 

573 # it is stored in the config. 

574 connections = 42 

575 with unittest.mock.patch.dict( 

576 os.environ, {"LSST_HTTP_BACKEND_PERSISTENT_CONNECTIONS": str(connections)}, clear=True 

577 ): 

578 importlib.reload(lsst.resources.http) 

579 config = HttpResourcePathConfig() 

580 self.assertTrue(config.back_end_connections, connections) 

581 

582 def test_digest_algorithm(self): 

583 # Ensure that when no digest is specified in the environment, the 

584 # configured digest algorithm is the empty string. 

585 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

586 importlib.reload(lsst.resources.http) 

587 config = HttpResourcePathConfig() 

588 self.assertEqual(config.digest_algorithm, "") 

589 

590 # Ensure that an invalid digest algorithm is ignored. 

591 digest = "invalid" 

592 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_DIGEST": digest}, clear=True): 

593 importlib.reload(lsst.resources.http) 

594 config = HttpResourcePathConfig() 

595 self.assertEqual(config.digest_algorithm, "") 

596 

597 # Ensure that an accepted digest algorithm is stored. 

598 for digest in lsst.resources.http.ACCEPTED_DIGESTS: 

599 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_DIGEST": digest}, clear=True): 

600 importlib.reload(lsst.resources.http) 

601 config = HttpResourcePathConfig() 

602 self.assertTrue(config.digest_algorithm, digest) 

603 

604 

605class WebdavUtilsTestCase(unittest.TestCase): 

606 """Test for the Webdav related utilities.""" 

607 

608 def setUp(self): 

609 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

610 

611 def tearDown(self): 

612 if self.tmpdir: 

613 if self.tmpdir.isLocal: 

614 removeTestTempDir(self.tmpdir.ospath) 

615 

616 @responses.activate 

617 def test_is_webdav_endpoint(self): 

618 davEndpoint = "http://www.lsstwithwebdav.org" 

619 responses.add(responses.OPTIONS, davEndpoint, status=200, headers={"DAV": "1,2,3"}) 

620 self.assertTrue(_is_webdav_endpoint(davEndpoint)) 

621 

622 plainHttpEndpoint = "http://www.lsstwithoutwebdav.org" 

623 responses.add(responses.OPTIONS, plainHttpEndpoint, status=200) 

624 self.assertFalse(_is_webdav_endpoint(plainHttpEndpoint)) 

625 

626 def test_is_protected(self): 

627 self.assertFalse(_is_protected("/this-file-does-not-exist")) 

628 

629 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

630 f.write("XXXX") 

631 file_path = f.name 

632 

633 os.chmod(file_path, stat.S_IRUSR) 

634 self.assertTrue(_is_protected(file_path)) 

635 

636 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

637 os.chmod(file_path, stat.S_IRUSR | mode) 

638 self.assertFalse(_is_protected(file_path)) 

639 

640 

641class BearerTokenAuthTestCase(unittest.TestCase): 

642 """Test for the BearerTokenAuth class.""" 

643 

644 def setUp(self): 

645 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

646 self.token = "ABCDE1234" 

647 

648 def tearDown(self): 

649 if self.tmpdir and self.tmpdir.isLocal: 

650 removeTestTempDir(self.tmpdir.ospath) 

651 

652 def test_empty_token(self): 

653 """Ensure that when no token is provided the request is not 

654 modified. 

655 """ 

656 auth = BearerTokenAuth(None) 

657 auth._refresh() 

658 self.assertIsNone(auth._token) 

659 self.assertIsNone(auth._path) 

660 req = requests.Request("GET", "https://example.org") 

661 self.assertEqual(auth(req), req) 

662 

663 def test_token_value(self): 

664 """Ensure that when a token value is provided, the 'Authorization' 

665 header is added to the requests. 

666 """ 

667 auth = BearerTokenAuth(self.token) 

668 req = auth(requests.Request("GET", "https://example.org").prepare()) 

669 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

670 

671 def test_token_file(self): 

672 """Ensure when the provided token is a file path, its contents is 

673 correctly used in the the 'Authorization' header of the requests. 

674 """ 

675 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

676 f.write(self.token) 

677 token_file_path = f.name 

678 

679 # Ensure the request's "Authorization" header is set with the right 

680 # token value 

681 os.chmod(token_file_path, stat.S_IRUSR) 

682 auth = BearerTokenAuth(token_file_path) 

683 req = auth(requests.Request("GET", "https://example.org").prepare()) 

684 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

685 

686 # Ensure an exception is raised if either group or other can read the 

687 # token file 

688 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

689 os.chmod(token_file_path, stat.S_IRUSR | mode) 

690 with self.assertRaises(PermissionError): 

691 BearerTokenAuth(token_file_path) 

692 

693 

694class SessionStoreTestCase(unittest.TestCase): 

695 """Test for the SessionStore class.""" 

696 

697 def setUp(self): 

698 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

699 self.rpath = ResourcePath("https://example.org") 

700 

701 def tearDown(self): 

702 if self.tmpdir and self.tmpdir.isLocal: 

703 removeTestTempDir(self.tmpdir.ospath) 

704 

705 def test_ca_cert_bundle(self): 

706 """Ensure a certificate authorities bundle is used to authentify 

707 the remote server. 

708 """ 

709 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

710 f.write("CERT BUNDLE") 

711 cert_bundle = f.name 

712 

713 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True): 

714 session = SessionStore().get(self.rpath) 

715 self.assertEqual(session.verify, cert_bundle) 

716 

717 def test_user_cert(self): 

718 """Ensure if user certificate and private key are provided, they are 

719 used for authenticating the client. 

720 """ 

721 

722 # Create mock certificate and private key files. 

723 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

724 f.write("CERT") 

725 client_cert = f.name 

726 

727 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

728 f.write("KEY") 

729 client_key = f.name 

730 

731 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY 

732 # must be initialized. 

733 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True): 

734 with self.assertRaises(ValueError): 

735 SessionStore().get(self.rpath) 

736 

737 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True): 

738 with self.assertRaises(ValueError): 

739 SessionStore().get(self.rpath) 

740 

741 # Check private key file must be accessible only by its owner. 

742 with unittest.mock.patch.dict( 

743 os.environ, 

744 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key}, 

745 clear=True, 

746 ): 

747 # Ensure the session client certificate is initialized when 

748 # only the owner can read the private key file. 

749 os.chmod(client_key, stat.S_IRUSR) 

750 session = SessionStore().get(self.rpath) 

751 self.assertEqual(session.cert[0], client_cert) 

752 self.assertEqual(session.cert[1], client_key) 

753 

754 # Ensure an exception is raised if either group or other can access 

755 # the private key file. 

756 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

757 os.chmod(client_key, stat.S_IRUSR | mode) 

758 with self.assertRaises(PermissionError): 

759 SessionStore().get(self.rpath) 

760 

761 def test_token_env(self): 

762 """Ensure when the token is provided via an environment variable 

763 the sessions are equipped with a BearerTokenAuth. 

764 """ 

765 token = "ABCDE" 

766 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True): 

767 session = SessionStore().get(self.rpath) 

768 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth) 

769 self.assertEqual(session.auth._token, token) 

770 self.assertIsNone(session.auth._path) 

771 

772 def test_sessions(self): 

773 """Ensure the session caching mechanism works.""" 

774 

775 # Ensure the store provides a session for a given URL 

776 root_url = "https://example.org" 

777 store = SessionStore() 

778 session = store.get(ResourcePath(root_url)) 

779 self.assertIsNotNone(session) 

780 

781 # Ensure the sessions retrieved from a single store with the same 

782 # root URIs are equal 

783 for u in (f"{root_url}", f"{root_url}/path/to/file"): 

784 self.assertEqual(session, store.get(ResourcePath(u))) 

785 

786 # Ensure sessions retrieved for different root URIs are different 

787 another_url = "https://another.example.org" 

788 self.assertNotEqual(session, store.get(ResourcePath(another_url))) 

789 

790 # Ensure the sessions retrieved from a single store for URLs with 

791 # different port numbers are different 

792 root_url_with_port = f"{another_url}:12345" 

793 session = store.get(ResourcePath(root_url_with_port)) 

794 self.assertNotEqual(session, store.get(ResourcePath(another_url))) 

795 

796 # Ensure the sessions retrieved from a single store with the same 

797 # root URIs (including port numbers) are equal 

798 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"): 

799 self.assertEqual(session, store.get(ResourcePath(u))) 

800 

801 

802if __name__ == "__main__": 802 ↛ 803line 802 didn't jump to line 803, because the condition on line 802 was never true

803 unittest.main()