Coverage for tests/test_http.py: 16%

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import hashlib 

13import io 

14import os.path 

15import pickle 

16import random 

17import shutil 

18import socket 

19import stat 

20import string 

21import tempfile 

22import time 

23import unittest 

24import unittest.mock 

25import warnings 

26from collections.abc import Callable 

27from threading import Thread 

28from typing import cast 

29 

30try: 

31 from cheroot import wsgi 

32 from wsgidav.wsgidav_app import WsgiDAVApp 

33except ImportError: 

34 WsgiDAVApp = None 

35 

36import requests 

37import responses 

38import responses.matchers 

39 

40import lsst.resources 

41from lsst.resources import ResourcePath 

42from lsst.resources._resourceHandles._httpResourceHandle import ( 

43 HttpReadResourceHandle, 

44 parse_content_range_header, 

45) 

46from lsst.resources.http import ( 

47 BearerTokenAuth, 

48 HttpResourcePath, 

49 HttpResourcePathConfig, 

50 SessionStore, 

51 _is_protected, 

52) 

53from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

54from lsst.resources.utils import _get_num_workers, makeTestTempDir, removeTestTempDir 

55 

56TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

57 

58 

59class GenericHttpTestCase(GenericTestCase, unittest.TestCase): 

60 """Generic tests of http URIs.""" 

61 

62 scheme = "http" 

63 netloc = "server.example" 

64 

65 def test_root_uri(self): 

66 self.assertEqual(ResourcePath("http://server.com").root_uri(), ResourcePath("http://server.com/")) 

67 self.assertEqual( 

68 ResourcePath("http://user:password@server.com:3000/").root_uri(), 

69 ResourcePath("http://user:password@server.com:3000/"), 

70 ) 

71 self.assertEqual( 

72 ResourcePath("http://user:password@server.com:3000/some/path").root_uri(), 

73 ResourcePath("http://user:password@server.com:3000/"), 

74 ) 

75 self.assertEqual( 

76 ResourcePath("http://user:password@server.com:3000/some/path#fragment").root_uri(), 

77 ResourcePath("http://user:password@server.com:3000/"), 

78 ) 

79 self.assertEqual( 

80 ResourcePath("http://user:password@server.com:3000/some/path?param=value").root_uri(), 

81 ResourcePath("http://user:password@server.com:3000/"), 

82 ) 

83 self.assertEqual( 

84 ResourcePath("http://user:password@server.com:3000/some/path;parameters").root_uri(), 

85 ResourcePath("http://user:password@server.com:3000/"), 

86 ) 

87 

88 @responses.activate 

89 def test_extra_headers(self): 

90 url = "http://test.example/something.txt" 

91 path = HttpResourcePath.create_http_resource_path( 

92 url, extra_headers={"Authorization": "Bearer my-token"} 

93 ) 

94 

95 self.assertEqual(str(path), "http://test.example/something.txt") 

96 self.assertEqual(path._extra_headers, {"Authorization": "Bearer my-token"}) 

97 

98 # Make sure that headers are added to requests. 

99 responses.add( 

100 responses.GET, 

101 url, 

102 b"test", 

103 match=[responses.matchers.header_matcher({"Authorization": "Bearer my-token"})], 

104 ) 

105 self.assertEqual(path.read(), b"test") 

106 

107 # Make sure that headers are added to fsspec. 

108 # This triggers logic for "webdav" vs "not-webdav" that does an OPTIONS 

109 # request, so we need to check that too. 

110 responses.add( 

111 responses.OPTIONS, 

112 "http://test.example/", 

113 match=[responses.matchers.header_matcher({"Authorization": "Bearer my-token"})], 

114 ) 

115 fs, _ = path.to_fsspec() 

116 self.assertEqual(fs.client_kwargs.get("headers"), {"Authorization": "Bearer my-token"}) 

117 

118 # Extra headers should be preserved through pickle, to ensure that 

119 # `mtransfer` and similar methods work in multi-process mode. 

120 dump = pickle.dumps(path) 

121 restored = pickle.loads(dump) 

122 self.assertEqual(restored._extra_headers, {"Authorization": "Bearer my-token"}) 

123 

124 # Extra headers should be preserved when making a modified copy of the 

125 # ResourcePath using replace() or the ResourcePath constructor. 

126 replacement = path.replace(forceDirectory=True) 

127 self.assertEqual(replacement._extra_headers, {"Authorization": "Bearer my-token"}) 

128 copy = ResourcePath(path, forceDirectory=True) 

129 self.assertEqual(copy._extra_headers, {"Authorization": "Bearer my-token"}) 

130 

131 

132class HttpReadWriteWebdavTestCase(GenericReadWriteTestCase, unittest.TestCase): 

133 """Test with a real webDAV server, as opposed to mocking responses.""" 

134 

135 scheme = "http" 

136 local_files_to_remove: list[str] = [] 

137 

138 @classmethod 

139 def setUpClass(cls): 

140 cls.webdav_tmpdir = tempfile.mkdtemp(prefix="webdav-server-test-") 

141 cls.server_thread = None 

142 

143 # Disable warnings about socket connections left open. We purposedly 

144 # keep network connections to the remote server open and have no 

145 # means through the API exposed by Requests of actually close the 

146 # underlyng sockets to make tests pass without warning. 

147 warnings.filterwarnings(action="ignore", message=r"unclosed.*socket", category=ResourceWarning) 

148 

149 # Should we test against a running server? 

150 # 

151 # This is convenient for testing against real servers in the 

152 # developer environment by initializing the environment variable 

153 # LSST_RESOURCES_HTTP_TEST_SERVER_URL with the URL of the server, e.g. 

154 # https://dav.example.org:1234/path/to/top/dir 

155 if (test_endpoint := os.getenv("LSST_RESOURCES_HTTP_TEST_SERVER_URL")) is not None: 

156 # Run this test case against the specified server. 

157 uri = ResourcePath(test_endpoint) 

158 cls.scheme = uri.scheme 

159 cls.netloc = uri.netloc 

160 cls.base_path = uri.path 

161 elif WsgiDAVApp is not None: 

162 # WsgiDAVApp is available, launch a local server in its own 

163 # thread to expose a local temporary directory and run this 

164 # test case against it. 

165 cls.port_number = cls._get_port_number() 

166 cls.stop_webdav_server = False 

167 cls.server_thread = Thread( 

168 target=cls._serve_webdav, 

169 args=(cls, cls.webdav_tmpdir, cls.port_number, lambda: cls.stop_webdav_server), 

170 daemon=True, 

171 ) 

172 cls.server_thread.start() 

173 

174 # Wait for it to start 

175 time.sleep(1) 

176 

177 # Initialize the server endpoint 

178 cls.netloc = f"127.0.0.1:{cls.port_number}" 

179 else: 

180 cls.skipTest( 

181 cls, 

182 "neither WsgiDAVApp is available nor a webDAV test endpoint is configured to test against", 

183 ) 

184 

185 @classmethod 

186 def tearDownClass(cls): 

187 # Stop the WsgiDAVApp server, if any 

188 if WsgiDAVApp is not None: 

189 # Shut down of the webdav server and wait for the thread to exit 

190 cls.stop_webdav_server = True 

191 if cls.server_thread is not None: 

192 cls.server_thread.join() 

193 

194 # Remove local temporary files 

195 for file in cls.local_files_to_remove: 

196 if os.path.exists(file): 

197 os.remove(file) 

198 

199 # Remove temp dir 

200 if cls.webdav_tmpdir: 

201 shutil.rmtree(cls.webdav_tmpdir, ignore_errors=True) 

202 

203 # Reset the warnings filter. 

204 warnings.resetwarnings() 

205 

206 def tearDown(self): 

207 if self.tmpdir: 

208 self.tmpdir.remove() 

209 

210 # Clear sessions. Some sockets may be left open, because urllib3 

211 # doest not close in-flight connections. 

212 # See https://urllib3.readthedocs.io > API Reference > 

213 # Pool Manager > clear() 

214 # I cannot add the full URL here because it is longer than 79 

215 # characters. 

216 self.tmpdir._clear_sessions() 

217 

218 super().tearDown() 

219 

220 def test_dav_file_handle(self): 

221 # Upload a new file with known contents. 

222 contents = "These are some \n bytes to read" 

223 remote_file = self.tmpdir.join(self._get_file_name()) 

224 self.assertIsNone(remote_file.write(data=contents, overwrite=True)) 

225 

226 # Test that the correct handle is returned. 

227 with remote_file.open("rb") as handle: 

228 self.assertIsInstance(handle, HttpReadResourceHandle) 

229 

230 # Test reading byte ranges works 

231 with remote_file.open("rb") as handle: 

232 sub_contents = contents[:10] 

233 handle = cast(HttpReadResourceHandle, handle) 

234 result = handle.read(len(sub_contents)).decode() 

235 self.assertEqual(result, sub_contents) 

236 # Verify there is no internal buffer. 

237 self.assertIsNone(handle._completeBuffer) 

238 # Verify the position. 

239 self.assertEqual(handle.tell(), len(sub_contents)) 

240 

241 # Jump back to the beginning and test if reading the whole file 

242 # prompts the internal buffer to be read. 

243 handle.seek(0) 

244 self.assertEqual(handle.tell(), 0) 

245 result = handle.read().decode() 

246 self.assertIsNotNone(handle._completeBuffer) 

247 self.assertEqual(result, contents) 

248 

249 # Check that flush works on read-only handle. 

250 handle.flush() 

251 

252 # Verify reading as a string handle works as expected. 

253 with remote_file.open("r") as handle: 

254 self.assertIsInstance(handle, io.TextIOWrapper) 

255 

256 handle = cast(io.TextIOWrapper, handle) 

257 self.assertIsInstance(handle.buffer, HttpReadResourceHandle) 

258 

259 # Check if string methods work. 

260 result = handle.read() 

261 self.assertEqual(result, contents) 

262 

263 # Check that flush works on read-only handle. 

264 handle.flush() 

265 

266 # Verify that write modes invoke the default base method 

267 with remote_file.open("w") as handle: 

268 self.assertIsInstance(handle, io.StringIO) 

269 

270 def test_dav_is_dav_enpoint(self): 

271 # Ensure the server is a webDAV endpoint 

272 self.assertTrue(self.tmpdir.is_webdav_endpoint) 

273 

274 def test_dav_mkdir(self): 

275 # Check creation and deletion of an empty directory 

276 subdir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

277 self.assertIsNone(subdir.mkdir()) 

278 self.assertTrue(subdir.exists()) 

279 

280 # Creating an existing remote directory must succeed 

281 self.assertIsNone(subdir.mkdir()) 

282 

283 # Deletion of an existing directory must succeed 

284 self.assertIsNone(subdir.remove()) 

285 

286 # Deletion of an non-existing directory must succeed 

287 subir_not_exists = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

288 self.assertIsNone(subir_not_exists.remove()) 

289 

290 # Creation of a directory at a path where a file exists must raise 

291 file = self.tmpdir.join(self._get_file_name(), forceDirectory=False) 

292 file.write(data=None, overwrite=True) 

293 self.assertTrue(file.exists()) 

294 

295 existing_file = self.tmpdir.join(file.basename(), forceDirectory=True) 

296 with self.assertRaises(NotADirectoryError): 

297 self.assertIsNone(existing_file.mkdir()) 

298 

299 def test_dav_upload_download(self): 

300 # Test upload a randomly-generated file via write() with and without 

301 # overwrite 

302 local_file, file_size = self._generate_file() 

303 with open(local_file, "rb") as f: 

304 data = f.read() 

305 

306 remote_file = self.tmpdir.join(self._get_file_name()) 

307 self.assertIsNone(remote_file.write(data, overwrite=True)) 

308 self.assertTrue(remote_file.exists()) 

309 self.assertEqual(remote_file.size(), file_size) 

310 

311 # Write without overwrite must raise since target file exists 

312 with self.assertRaises(FileExistsError): 

313 remote_file.write(data, overwrite=False) 

314 

315 # Download the file we just uploaded. Compute and compare a digest of 

316 # the uploaded and downloaded data and ensure they match 

317 downloaded_data = remote_file.read() 

318 self.assertEqual(len(downloaded_data), file_size) 

319 upload_digest = self._compute_digest(data) 

320 download_digest = self._compute_digest(downloaded_data) 

321 self.assertEqual(upload_digest, download_digest) 

322 os.remove(local_file) 

323 

324 def test_dav_as_local(self): 

325 contents = str.encode("12345") 

326 remote_file = self.tmpdir.join(self._get_file_name()) 

327 self.assertIsNone(remote_file.write(data=contents, overwrite=True)) 

328 

329 with remote_file._as_local() as local_uri: 

330 self.assertTrue(local_uri.isTemporary) 

331 self.assertTrue(os.path.exists(local_uri.ospath)) 

332 self.assertTrue(os.stat(local_uri.ospath).st_size, len(contents)) 

333 self.assertEqual(local_uri.read(), contents) 

334 self.assertFalse(local_uri.exists()) 

335 

336 def test_dav_size(self): 

337 # Size of a non-existent file must raise. 

338 remote_file = self.tmpdir.join(self._get_file_name()) 

339 with self.assertRaises(FileNotFoundError): 

340 remote_file.size() 

341 

342 # Retrieving the size of a remote directory using a file-like path must 

343 # raise 

344 remote_dir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

345 self.assertIsNone(remote_dir.mkdir()) 

346 self.assertTrue(remote_dir.exists()) 

347 

348 dir_as_file = ResourcePath(remote_dir.geturl().rstrip("/"), forceDirectory=False) 

349 with self.assertRaises(IsADirectoryError): 

350 dir_as_file.size() 

351 

352 def test_dav_upload_creates_dir(self): 

353 # Uploading a file to a non existing directory must ensure its 

354 # parent directories are automatically created and upload succeeds 

355 non_existing_dir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

356 non_existing_dir = non_existing_dir.join(self._get_dir_name(), forceDirectory=True) 

357 non_existing_dir = non_existing_dir.join(self._get_dir_name(), forceDirectory=True) 

358 remote_file = non_existing_dir.join(self._get_file_name()) 

359 

360 local_file, file_size = self._generate_file() 

361 with open(local_file, "rb") as f: 

362 data = f.read() 

363 self.assertIsNone(remote_file.write(data, overwrite=True)) 

364 

365 self.assertTrue(remote_file.exists()) 

366 self.assertEqual(remote_file.size(), file_size) 

367 self.assertTrue(remote_file.parent().exists()) 

368 

369 downloaded_data = remote_file.read() 

370 upload_digest = self._compute_digest(data) 

371 download_digest = self._compute_digest(downloaded_data) 

372 self.assertEqual(upload_digest, download_digest) 

373 os.remove(local_file) 

374 

375 def test_dav_transfer_from(self): 

376 # Transfer from local file via "copy", with and without overwrite 

377 remote_file = self.tmpdir.join(self._get_file_name()) 

378 local_file, _ = self._generate_file() 

379 source_file = ResourcePath(local_file) 

380 self.assertIsNone(remote_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

381 self.assertTrue(remote_file.exists()) 

382 self.assertEqual(remote_file.size(), source_file.size()) 

383 with self.assertRaises(FileExistsError): 

384 remote_file.transfer_from(ResourcePath(local_file), transfer="copy", overwrite=False) 

385 

386 # Transfer from remote file via "copy", with and without overwrite 

387 source_file = remote_file 

388 target_file = self.tmpdir.join(self._get_file_name()) 

389 self.assertIsNone(target_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

390 self.assertTrue(target_file.exists()) 

391 self.assertEqual(target_file.size(), source_file.size()) 

392 

393 # Transfer without overwrite must raise since target resource exists 

394 with self.assertRaises(FileExistsError): 

395 target_file.transfer_from(source_file, transfer="copy", overwrite=False) 

396 

397 # Test transfer from local file via "move", with and without overwrite 

398 source_file = ResourcePath(local_file) 

399 source_size = source_file.size() 

400 target_file = self.tmpdir.join(self._get_file_name()) 

401 self.assertIsNone(target_file.transfer_from(source_file, transfer="move", overwrite=True)) 

402 self.assertTrue(target_file.exists()) 

403 self.assertEqual(target_file.size(), source_size) 

404 self.assertFalse(source_file.exists()) 

405 

406 # Test transfer without overwrite must raise since target resource 

407 # exists 

408 local_file, file_size = self._generate_file() 

409 with self.assertRaises(FileExistsError): 

410 source_file = ResourcePath(local_file) 

411 target_file.transfer_from(source_file, transfer="move", overwrite=False) 

412 

413 # Test transfer from remote file via "move" with and without overwrite 

414 # must succeed 

415 source_file = target_file 

416 source_size = source_file.size() 

417 target_file = self.tmpdir.join(self._get_file_name()) 

418 self.assertIsNone(target_file.transfer_from(source_file, transfer="move", overwrite=True)) 

419 self.assertTrue(target_file.exists()) 

420 self.assertEqual(target_file.size(), source_size) 

421 self.assertFalse(source_file.exists()) 

422 

423 # Transfer without overwrite must raise since target resource exists 

424 with self.assertRaises(FileExistsError): 

425 source_file = ResourcePath(local_file) 

426 target_file.transfer_from(source_file, transfer="move", overwrite=False) 

427 

428 def test_dav_handle(self): 

429 # Resource handle must succeed 

430 target_file = self.tmpdir.join(self._get_file_name()) 

431 data = "abcdefghi" 

432 self.assertIsNone(target_file.write(data, overwrite=True)) 

433 with target_file.open("rb") as handle: 

434 handle.seek(1) 

435 self.assertEqual(handle.read(4).decode("utf-8"), data[1:5]) 

436 

437 def test_dav_delete(self): 

438 # Deletion of an existing remote file must succeed 

439 local_file, file_size = self._generate_file() 

440 with open(local_file, "rb") as f: 

441 data = f.read() 

442 

443 remote_file = self.tmpdir.join(self._get_file_name()) 

444 self.assertIsNone(remote_file.write(data, overwrite=True)) 

445 self.assertTrue(remote_file.exists()) 

446 self.assertEqual(remote_file.size(), file_size) 

447 self.assertIsNone(remote_file.remove()) 

448 os.remove(local_file) 

449 

450 # Deletion of a non-existing remote file must succeed 

451 non_existing_file = self.tmpdir.join(self._get_file_name()) 

452 self.assertIsNone(non_existing_file.remove()) 

453 

454 # Deletion of a non-empty remote directory must succeed 

455 subdir = self.tmpdir.join(self._get_dir_name(), forceDirectory=True) 

456 self.assertIsNone(subdir.mkdir()) 

457 self.assertTrue(subdir.exists()) 

458 local_file, _ = self._generate_file() 

459 source_file = ResourcePath(local_file) 

460 target_file = self.tmpdir.join(self._get_file_name(), forceDirectory=True) 

461 self.assertIsNone(target_file.transfer_from(source_file, transfer="copy", overwrite=True)) 

462 self.assertIsNone(subdir.remove()) 

463 self.assertFalse(subdir.exists()) 

464 os.remove(local_file) 

465 

466 def test_dav_to_fsspec(self): 

467 # Upload a randomly-generated file via write() with overwrite. 

468 local_file, file_size = self._generate_file() 

469 with open(local_file, "rb") as f: 

470 data = f.read() 

471 

472 remote_file = self.tmpdir.join(self._get_file_name()) 

473 self.assertIsNone(remote_file.write(data, overwrite=True)) 

474 self.assertTrue(remote_file.exists()) 

475 self.assertEqual(remote_file.size(), file_size) 

476 remote_file_url = remote_file.geturl() 

477 

478 # to_fsspec() may raise if that feature is not specifically 

479 # enabled in the environment and remote server is one of the 

480 # webDAV servers that support signing URLs. 

481 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

482 try: 

483 # Force reinitialization of the config from the environment 

484 HttpResourcePath._reload_config() 

485 fsys, url = ResourcePath(remote_file_url).to_fsspec() 

486 self.assertEqual(data, fsys.cat(url)) 

487 except ImportError as e: 

488 self.assertTrue("disable" in str(e)) 

489 

490 # Ensure to_fsspec() works if that feature is enabled in the 

491 # environment. 

492 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_ENABLE_FSSPEC": "true"}, clear=True): 

493 try: 

494 # Force reinitialization of the config from the environment. 

495 HttpResourcePath._reload_config() 

496 rpath = ResourcePath(remote_file_url) 

497 

498 # Ensure that the contents of the remote file we just 

499 # uploaded is identical to the contents of that file when 

500 # retrieved via fsspec.open(). 

501 fsys, url = rpath.to_fsspec() 

502 with fsys.open(url) as f: 

503 self.assertEqual(data, f.read()) 

504 

505 # Ensure the contents is identical to the result of 

506 # fsspec.cat() 

507 self.assertEqual(data, fsys.cat(url)) 

508 

509 # Ensure that attempting to modify a remote via via fsspec 

510 # fails, since the returned URL is signed for download only. 

511 # fsspec.rm() raises NotImplementedError if it cannot remove 

512 # the remote file. 

513 if rpath.server_signs_urls: 

514 with self.assertRaises(NotImplementedError): 

515 fsys, url = rpath.to_fsspec() 

516 fsys.rm(url) 

517 except NotImplementedError as e: 

518 # to_fsspec() must succeed if remote server knows how to 

519 # sign URLs 

520 if rpath.server_signs_urls: 

521 raise e 

522 

523 # Force reinitialization of the config from the environment and 

524 # clean up local file. 

525 HttpResourcePath._reload_config() 

526 os.remove(local_file) 

527 

528 @responses.activate 

529 def test_is_webdav_endpoint(self): 

530 davEndpoint = "http://www.lsstwithwebdav.org" 

531 responses.add(responses.OPTIONS, davEndpoint, status=200, headers={"DAV": "1,2,3"}) 

532 self.assertTrue(ResourcePath(davEndpoint).is_webdav_endpoint) 

533 

534 plainHttpEndpoint = "http://www.lsstwithoutwebdav.org" 

535 responses.add(responses.OPTIONS, plainHttpEndpoint, status=200) 

536 self.assertFalse(ResourcePath(plainHttpEndpoint).is_webdav_endpoint) 

537 

538 notWebdavEndpoint = "http://www.notwebdav.org" 

539 responses.add(responses.OPTIONS, notWebdavEndpoint, status=403) 

540 self.assertFalse(ResourcePath(notWebdavEndpoint).is_webdav_endpoint) 

541 

542 @responses.activate 

543 def test_plain_http_url_signing(self): 

544 # As in test_is_webdav_endpoint above, configure a URL to appear as a 

545 # non-webdav HTTP server. 

546 plainHttpEndpoint = "http://nonwebdav.test" 

547 responses.add(responses.OPTIONS, plainHttpEndpoint, status=200) 

548 

549 # Plain HTTP URLs are already readable without authentication, so 

550 # generating a pre-signed URL is a no-op. 

551 path = ResourcePath("http://nonwebdav.test/file#frag") 

552 self.assertEqual( 

553 path.generate_presigned_get_url(expiration_time_seconds=300), "http://nonwebdav.test/file#frag" 

554 ) 

555 

556 # Writing to an arbitrary plain HTTP URL is unlikely to work, so we 

557 # don't generate put URLs. 

558 with self.assertRaises(NotImplementedError): 

559 path.generate_presigned_put_url(expiration_time_seconds=300) 

560 

561 @responses.activate 

562 def test_server_identity(self): 

563 server = "MyServer/v1.2.3" 

564 endpointWithServer = "http://www.lsstwithserverheader.org" 

565 responses.add(responses.OPTIONS, endpointWithServer, status=200, headers={"Server": server}) 

566 self.assertEqual(ResourcePath(endpointWithServer).server, "myserver") 

567 

568 endpointWithoutServer = "http://www.lsstwithoutserverheader.org" 

569 responses.add(responses.OPTIONS, endpointWithoutServer, status=200) 

570 self.assertIsNone(ResourcePath(endpointWithoutServer).server) 

571 

572 @classmethod 

573 def _get_port_number(cls) -> int: 

574 """Return a port number the webDAV server can use to listen to.""" 

575 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 

576 s.bind(("127.0.0.1", 0)) 

577 s.listen() 

578 port = s.getsockname()[1] 

579 s.close() 

580 return port 

581 

582 def _serve_webdav(self, local_path: str, port: int, stop_webdav_server: Callable[[], bool]): 

583 """Start a local webDAV server, listening on http://localhost:port 

584 and exposing local_path. 

585 

586 This server only runs when this test class is instantiated, 

587 and then shuts down. The server must be started is a separate thread. 

588 

589 Parameters 

590 ---------- 

591 port : `int` 

592 The port number on which the server should listen 

593 local_path : `str` 

594 Path to an existing local directory for the server to expose. 

595 stop_webdav_server : `Callable[[], bool]` 

596 Boolean function which returns True when the server should be 

597 stopped. 

598 """ 

599 try: 

600 # Start the wsgi server in a separate thread 

601 config = { 

602 "host": "127.0.0.1", 

603 "port": port, 

604 "provider_mapping": {"/": local_path}, 

605 "http_authenticator": {"domain_controller": None}, 

606 "simple_dc": {"user_mapping": {"*": True}}, 

607 "verbose": 0, 

608 "lock_storage": False, 

609 "dir_browser": { 

610 "enable": False, 

611 "ms_sharepoint_support": False, 

612 "libre_office_support": False, 

613 "response_trailer": False, 

614 "davmount_links": False, 

615 }, 

616 } 

617 server = wsgi.Server(wsgi_app=WsgiDAVApp(config), bind_addr=(config["host"], config["port"])) 

618 t = Thread(target=server.start, daemon=True) 

619 t.start() 

620 

621 # Shut down the server when done: stop_webdav_server() returns 

622 # True when this test suite is being teared down 

623 while not stop_webdav_server(): 

624 time.sleep(1) 

625 except KeyboardInterrupt: 

626 # Caught Ctrl-C, shut down the server 

627 pass 

628 finally: 

629 server.stop() 

630 t.join() 

631 

632 @classmethod 

633 def _get_name(cls, prefix: str) -> str: 

634 alphabet = string.ascii_lowercase + string.digits 

635 return f"{prefix}-" + "".join(random.choices(alphabet, k=8)) 

636 

637 @classmethod 

638 def _get_dir_name(cls) -> str: 

639 """Return a randomly selected name for a file""" 

640 return cls._get_name(prefix="dir") 

641 

642 @classmethod 

643 def _get_file_name(cls) -> str: 

644 """Return a randomly selected name for a file""" 

645 return cls._get_name(prefix="file") 

646 

647 def _generate_file(self, remove_when_done=True) -> tuple[str, int]: 

648 """Create a local file of random size with random contents. 

649 

650 Returns 

651 ------- 

652 path : `str` 

653 Path to local temporary file. The caller is responsible for 

654 removing the file when appropriate. 

655 size : `int` 

656 Size of the generated file, in bytes. 

657 """ 

658 megabyte = 1024 * 1024 

659 size = random.randint(2 * megabyte, 5 * megabyte) 

660 tmpfile, path = tempfile.mkstemp() 

661 self.assertEqual(os.write(tmpfile, os.urandom(size)), size) 

662 os.close(tmpfile) 

663 

664 if remove_when_done: 

665 HttpReadWriteWebdavTestCase.local_files_to_remove.append(path) 

666 

667 return path, size 

668 

669 @classmethod 

670 def _compute_digest(cls, data: bytes) -> str: 

671 """Compute a SHA256 hash of data.""" 

672 m = hashlib.sha256() 

673 m.update(data) 

674 return m.hexdigest() 

675 

676 @classmethod 

677 def _is_server_running(cls, port: int) -> bool: 

678 """Return True if there is a server listening on local address 

679 127.0.0.1:<port>. 

680 """ 

681 with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 

682 try: 

683 s.connect(("127.0.0.1", port)) 

684 return True 

685 except ConnectionRefusedError: 

686 return False 

687 

688 

689class HttpResourcePathConfigTestCase(unittest.TestCase): 

690 """Test for the HttpResourcePathConfig class.""" 

691 

692 def setUp(self): 

693 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

694 

695 def tearDown(self): 

696 if self.tmpdir and self.tmpdir.isLocal: 

697 removeTestTempDir(self.tmpdir.ospath) 

698 

699 def test_send_expect_header(self): 

700 # Ensure environment variable LSST_HTTP_PUT_SEND_EXPECT_HEADER is 

701 # inspected to initialize the HttpResourcePathConfig class. 

702 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

703 config = HttpResourcePathConfig() 

704 self.assertFalse(config.send_expect_on_put) 

705 

706 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True): 

707 config = HttpResourcePathConfig() 

708 self.assertTrue(config.send_expect_on_put) 

709 

710 def test_enable_fsspec(self): 

711 # Ensure environment variable LSST_HTTP_ENABLE_FSSPEC is 

712 # inspected to initialize the HttpResourcePathConfig class. 

713 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

714 config = HttpResourcePathConfig() 

715 self.assertFalse(config.fsspec_is_enabled) 

716 

717 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_ENABLE_FSSPEC": "any value"}, clear=True): 

718 config = HttpResourcePathConfig() 

719 self.assertTrue(config.fsspec_is_enabled) 

720 

721 def test_collect_memory_usage(self): 

722 # Ensure environment variable LSST_HTTP_COLLECT_MEMORY_USAGE is 

723 # inspected to initialize the HttpResourcePathConfig class. 

724 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

725 config = HttpResourcePathConfig() 

726 self.assertFalse(config.collect_memory_usage) 

727 

728 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_COLLECT_MEMORY_USAGE": "true"}, clear=True): 

729 config = HttpResourcePathConfig() 

730 self.assertTrue(config.collect_memory_usage) 

731 

732 def test_timeout(self): 

733 # Ensure that when the connect and read timeouts are not specified 

734 # the default values are stored in the config. 

735 with unittest.mock.patch.dict(os.environ, {}, clear=True): 

736 config = HttpResourcePathConfig() 

737 self.assertAlmostEqual(config.timeout[0], config.DEFAULT_TIMEOUT_CONNECT) 

738 self.assertAlmostEqual(config.timeout[1], config.DEFAULT_TIMEOUT_READ) 

739 

740 # Ensure that when both the connect and read timeouts are specified 

741 # they are both stored in the config. 

742 connect_timeout, read_timeout = 100.5, 200.8 

743 with unittest.mock.patch.dict( 

744 os.environ, 

745 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)}, 

746 clear=True, 

747 ): 

748 config = HttpResourcePathConfig() 

749 self.assertAlmostEqual(config.timeout[0], connect_timeout) 

750 self.assertAlmostEqual(config.timeout[1], read_timeout) 

751 

752 # Ensure that invalid float values (including NaN values) raise a 

753 # ValueError. 

754 for value in ("invalid", "NaN"): 

755 with unittest.mock.patch.dict( 

756 os.environ, 

757 {"LSST_HTTP_TIMEOUT_CONNECT": value, "LSST_HTTP_TIMEOUT_READ": value}, 

758 clear=True, 

759 ): 

760 with self.assertRaises(ValueError): 

761 config = HttpResourcePathConfig()