Coverage for tests/test_http.py: 18%

245 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-15 00:05 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import importlib 

13import os.path 

14import stat 

15import tempfile 

16import unittest 

17 

18import lsst.resources 

19import requests 

20import responses 

21from lsst.resources import ResourcePath 

22from lsst.resources.http import BearerTokenAuth, SessionStore, _is_protected, _is_webdav_endpoint 

23from lsst.resources.tests import GenericTestCase 

24from lsst.resources.utils import makeTestTempDir, removeTestTempDir 

25 

26TESTDIR = os.path.abspath(os.path.dirname(__file__)) 

27 

28 

29class GenericHttpTestCase(GenericTestCase, unittest.TestCase): 

30 scheme = "http" 

31 netloc = "server.example" 

32 

33 

34class HttpReadWriteTestCase(unittest.TestCase): 

35 """Specialist test cases for WebDAV server. 

36 

37 The responses class requires that every possible request be explicitly 

38 mocked out. This currently makes it extremely inconvenient to subclass 

39 the generic read/write tests shared by other URI schemes. For now use 

40 explicit standalone tests. 

41 """ 

42 

43 def setUp(self): 

44 # Local test directory 

45 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

46 

47 serverRoot = "www.not-exists.orgx" 

48 existingFolderName = "existingFolder" 

49 existingFileName = "existingFile" 

50 notExistingFileName = "notExistingFile" 

51 

52 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True) 

53 self.existingFileResourcePath = ResourcePath( 

54 f"https://{serverRoot}/{existingFolderName}/{existingFileName}" 

55 ) 

56 self.notExistingFileResourcePath = ResourcePath( 

57 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}" 

58 ) 

59 self.existingFolderResourcePath = ResourcePath( 

60 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True 

61 ) 

62 self.notExistingFolderResourcePath = ResourcePath( 

63 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True 

64 ) 

65 

66 # Need to declare the options 

67 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"}) 

68 

69 # Used by HttpResourcePath.exists() 

70 responses.add( 

71 responses.HEAD, 

72 self.existingFileResourcePath.geturl(), 

73 status=200, 

74 headers={"Content-Length": "1024"}, 

75 ) 

76 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404) 

77 

78 # Used by HttpResourcePath.read() 

79 responses.add( 

80 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!") 

81 ) 

82 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404) 

83 

84 # Used by HttpResourcePath.write() 

85 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201) 

86 

87 # Used by HttpResourcePath.transfer_from() 

88 responses.add( 

89 responses.Response( 

90 url=self.existingFileResourcePath.geturl(), 

91 method="COPY", 

92 headers={"Destination": self.existingFileResourcePath.geturl()}, 

93 status=201, 

94 ) 

95 ) 

96 responses.add( 

97 responses.Response( 

98 url=self.existingFileResourcePath.geturl(), 

99 method="COPY", 

100 headers={"Destination": self.notExistingFileResourcePath.geturl()}, 

101 status=201, 

102 ) 

103 ) 

104 responses.add( 

105 responses.Response( 

106 url=self.existingFileResourcePath.geturl(), 

107 method="MOVE", 

108 headers={"Destination": self.notExistingFileResourcePath.geturl()}, 

109 status=201, 

110 ) 

111 ) 

112 

113 # Used by HttpResourcePath.remove() 

114 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200) 

115 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404) 

116 

117 # Used by HttpResourcePath.mkdir() 

118 responses.add( 

119 responses.HEAD, 

120 self.existingFolderResourcePath.geturl(), 

121 status=200, 

122 headers={"Content-Length": "1024"}, 

123 ) 

124 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"}) 

125 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404) 

126 responses.add( 

127 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201) 

128 ) 

129 responses.add( 

130 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403) 

131 ) 

132 

133 # Used by HttpResourcePath._do_put() 

134 self.redirectPathNoExpect = ResourcePath(f"https://{serverRoot}/redirect-no-expect/file") 

135 self.redirectPathExpect = ResourcePath(f"https://{serverRoot}/redirect-expect/file") 

136 redirected_url = f"https://{serverRoot}/redirect/location" 

137 responses.add( 

138 responses.PUT, 

139 self.redirectPathNoExpect.geturl(), 

140 headers={"Location": redirected_url}, 

141 status=307, 

142 ) 

143 responses.add( 

144 responses.PUT, 

145 self.redirectPathExpect.geturl(), 

146 headers={"Location": redirected_url}, 

147 status=307, 

148 match=[responses.matchers.header_matcher({"Content-Length": "0", "Expect": "100-continue"})], 

149 ) 

150 responses.add(responses.PUT, redirected_url, status=202) 

151 

152 def tearDown(self): 

153 if self.tmpdir: 

154 if self.tmpdir.isLocal: 

155 removeTestTempDir(self.tmpdir.ospath) 

156 

157 @responses.activate 

158 def test_exists(self): 

159 self.assertTrue(self.existingFileResourcePath.exists()) 

160 self.assertFalse(self.notExistingFileResourcePath.exists()) 

161 

162 self.assertEqual(self.existingFileResourcePath.size(), 1024) 

163 with self.assertRaises(FileNotFoundError): 

164 self.notExistingFileResourcePath.size() 

165 

166 @responses.activate 

167 def test_remove(self): 

168 self.assertIsNone(self.existingFileResourcePath.remove()) 

169 with self.assertRaises(FileNotFoundError): 

170 self.notExistingFileResourcePath.remove() 

171 

172 url = "https://example.org/delete" 

173 responses.add(responses.DELETE, url, status=404) 

174 with self.assertRaises(FileNotFoundError): 

175 ResourcePath(url).remove() 

176 

177 @responses.activate 

178 def test_mkdir(self): 

179 # The mock means that we can't check this now exists 

180 self.notExistingFolderResourcePath.mkdir() 

181 

182 # This should do nothing 

183 self.existingFolderResourcePath.mkdir() 

184 

185 with self.assertRaises(ValueError): 

186 self.notExistingFileResourcePath.mkdir() 

187 

188 @responses.activate 

189 def test_read(self): 

190 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!") 

191 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.") 

192 with self.assertRaises(FileNotFoundError): 

193 self.notExistingFileResourcePath.read() 

194 

195 # Run this twice to ensure use of cache in code coverage. 

196 for _ in (1, 2): 

197 with self.existingFileResourcePath.as_local() as local_uri: 

198 self.assertTrue(local_uri.isLocal) 

199 content = local_uri.read().decode() 

200 self.assertEqual(content, "It works!") 

201 

202 # Check that the environment variable is being read. 

203 lsst.resources.http._TMPDIR = None 

204 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir.ospath}): 

205 with self.existingFileResourcePath.as_local() as local_uri: 

206 self.assertTrue(local_uri.isLocal) 

207 content = local_uri.read().decode() 

208 self.assertEqual(content, "It works!") 

209 self.assertIsNotNone(local_uri.relative_to(self.tmpdir)) 

210 

211 @responses.activate 

212 def test_write(self): 

213 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content."))) 

214 with self.assertRaises(FileExistsError): 

215 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False) 

216 

217 url = "https://example.org/put" 

218 responses.add(responses.PUT, url, status=404) 

219 with self.assertRaises(ValueError): 

220 ResourcePath(url).write(data=str.encode("Some content.")) 

221 

222 @responses.activate 

223 def test_do_put_with_redirection(self): 

224 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER. 

225 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None) 

226 importlib.reload(lsst.resources.http) 

227 body = str.encode("any contents") 

228 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body)) 

229 

230 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER. 

231 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True): 

232 importlib.reload(lsst.resources.http) 

233 self.assertIsNone(self.redirectPathExpect._do_put(data=body)) 

234 

235 @responses.activate 

236 def test_transfer(self): 

237 # Transferring to self should be no-op. 

238 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath) 

239 

240 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)) 

241 # Should test for existence. 

242 # self.assertTrue(self.notExistingFileResourcePath.exists()) 

243 

244 # Should delete and try again with move. 

245 # self.notExistingFileResourcePath.remove() 

246 self.assertIsNone( 

247 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move") 

248 ) 

249 # Should then check that it was moved. 

250 # self.assertFalse(self.existingFileResourcePath.exists()) 

251 

252 # Existing file resource should have been removed so this should 

253 # trigger FileNotFoundError. 

254 # with self.assertRaises(FileNotFoundError): 

255 # self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath) 

256 with self.assertRaises(ValueError): 

257 self.notExistingFileResourcePath.transfer_from( 

258 src=self.existingFileResourcePath, transfer="unsupported" 

259 ) 

260 

261 def test_parent(self): 

262 self.assertEqual( 

263 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl() 

264 ) 

265 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl()) 

266 self.assertEqual( 

267 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl() 

268 ) 

269 

270 def test_send_expect_header(self): 

271 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from 

272 # the environment. 

273 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None) 

274 importlib.reload(lsst.resources.http) 

275 self.assertFalse(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT) 

276 

277 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True): 

278 importlib.reload(lsst.resources.http) 

279 self.assertTrue(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT) 

280 

281 def test_timeout(self): 

282 connect_timeout = 100 

283 read_timeout = 200 

284 with unittest.mock.patch.dict( 

285 os.environ, 

286 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)}, 

287 clear=True, 

288 ): 

289 # Force module reload to initialize TIMEOUT. 

290 importlib.reload(lsst.resources.http) 

291 self.assertEqual(lsst.resources.http.TIMEOUT, (connect_timeout, read_timeout)) 

292 

293 def test_is_protected(self): 

294 self.assertFalse(_is_protected("/this-file-does-not-exist")) 

295 

296 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

297 f.write("XXXX") 

298 file_path = f.name 

299 

300 os.chmod(file_path, stat.S_IRUSR) 

301 self.assertTrue(_is_protected(file_path)) 

302 

303 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

304 os.chmod(file_path, stat.S_IRUSR | mode) 

305 self.assertFalse(_is_protected(file_path)) 

306 

307 

308class WebdavUtilsTestCase(unittest.TestCase): 

309 """Test for the Webdav related utilities.""" 

310 

311 serverRoot = "www.lsstwithwebdav.orgx" 

312 wrongRoot = "www.lsstwithoutwebdav.org" 

313 

314 def setUp(self): 

315 responses.add(responses.OPTIONS, f"https://{self.serverRoot}", status=200, headers={"DAV": "1,2,3"}) 

316 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200) 

317 

318 @responses.activate 

319 def test_is_webdav_endpoint(self): 

320 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}")) 

321 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}")) 

322 

323 

324class BearerTokenAuthTestCase(unittest.TestCase): 

325 """Test for the BearerTokenAuth class.""" 

326 

327 def setUp(self): 

328 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

329 self.token = "ABCDE1234" 

330 

331 def tearDown(self): 

332 if self.tmpdir and self.tmpdir.isLocal: 

333 removeTestTempDir(self.tmpdir.ospath) 

334 

335 def test_empty_token(self): 

336 """Ensure that when no token is provided the request is not 

337 modified. 

338 """ 

339 auth = BearerTokenAuth(None) 

340 auth._refresh() 

341 self.assertIsNone(auth._token) 

342 self.assertIsNone(auth._path) 

343 req = requests.Request("GET", "https://example.org") 

344 self.assertEqual(auth(req), req) 

345 

346 def test_token_value(self): 

347 """Ensure that when a token value is provided, the 'Authorization' 

348 header is added to the requests. 

349 """ 

350 auth = BearerTokenAuth(self.token) 

351 req = auth(requests.Request("GET", "https://example.org").prepare()) 

352 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

353 

354 def test_token_file(self): 

355 """Ensure when the provided token is a file path, its contents is 

356 correctly used in the the 'Authorization' header of the requests. 

357 """ 

358 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

359 f.write(self.token) 

360 token_file_path = f.name 

361 

362 # Ensure the request's "Authorization" header is set with the right 

363 # token value 

364 os.chmod(token_file_path, stat.S_IRUSR) 

365 auth = BearerTokenAuth(token_file_path) 

366 req = auth(requests.Request("GET", "https://example.org").prepare()) 

367 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}") 

368 

369 # Ensure an exception is raised if either group or other can read the 

370 # token file 

371 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

372 os.chmod(token_file_path, stat.S_IRUSR | mode) 

373 with self.assertRaises(PermissionError): 

374 BearerTokenAuth(token_file_path) 

375 

376 

377class SessionStoreTestCase(unittest.TestCase): 

378 """Test for the SessionStore class.""" 

379 

380 def setUp(self): 

381 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR)) 

382 self.rpath = ResourcePath("https://example.org") 

383 

384 def tearDown(self): 

385 if self.tmpdir and self.tmpdir.isLocal: 

386 removeTestTempDir(self.tmpdir.ospath) 

387 

388 def test_ca_cert_bundle(self): 

389 """Ensure a certificate authorities bundle is used to authentify 

390 the remote server. 

391 """ 

392 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

393 f.write("CERT BUNDLE") 

394 cert_bundle = f.name 

395 

396 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True): 

397 session = SessionStore().get(self.rpath) 

398 self.assertEqual(session.verify, cert_bundle) 

399 

400 def test_user_cert(self): 

401 """Ensure if user certificate and private key are provided, they are 

402 used for authenticating the client. 

403 """ 

404 

405 # Create mock certificate and private key files. 

406 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

407 f.write("CERT") 

408 client_cert = f.name 

409 

410 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f: 

411 f.write("KEY") 

412 client_key = f.name 

413 

414 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY 

415 # must be initialized. 

416 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True): 

417 with self.assertRaises(ValueError): 

418 SessionStore().get(self.rpath) 

419 

420 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True): 

421 with self.assertRaises(ValueError): 

422 SessionStore().get(self.rpath) 

423 

424 # Check private key file must be accessible only by its owner. 

425 with unittest.mock.patch.dict( 

426 os.environ, 

427 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key}, 

428 clear=True, 

429 ): 

430 # Ensure the session client certificate is initialized when 

431 # only the owner can read the private key file. 

432 os.chmod(client_key, stat.S_IRUSR) 

433 session = SessionStore().get(self.rpath) 

434 self.assertEqual(session.cert[0], client_cert) 

435 self.assertEqual(session.cert[1], client_key) 

436 

437 # Ensure an exception is raised if either group or other can access 

438 # the private key file. 

439 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH): 

440 os.chmod(client_key, stat.S_IRUSR | mode) 

441 with self.assertRaises(PermissionError): 

442 SessionStore().get(self.rpath) 

443 

444 def test_token_env(self): 

445 """Ensure when the token is provided via an environment variable 

446 the sessions are equipped with a BearerTokenAuth. 

447 """ 

448 token = "ABCDE" 

449 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True): 

450 session = SessionStore().get(self.rpath) 

451 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth) 

452 self.assertEqual(session.auth._token, token) 

453 self.assertIsNone(session.auth._path) 

454 

455 def test_sessions(self): 

456 """Ensure the session caching mechanism works.""" 

457 

458 # Ensure the store provides a session for a given URL 

459 root_url = "https://example.org" 

460 store = SessionStore() 

461 session = store.get(ResourcePath(root_url)) 

462 self.assertIsNotNone(session) 

463 

464 # Ensure the sessions retrieved from a single store with the same 

465 # root URIs are equal 

466 for u in (f"{root_url}", f"{root_url}/path/to/file"): 

467 self.assertEqual(session, store.get(ResourcePath(u))) 

468 

469 # Ensure sessions retrieved for different root URIs are different 

470 another_url = "https://another.example.org" 

471 self.assertNotEqual(session, store.get(ResourcePath(another_url))) 

472 

473 # Ensure the sessions retrieved from a single store for URLs with 

474 # different port numbers are different 

475 root_url_with_port = f"{another_url}:12345" 

476 session = store.get(ResourcePath(root_url_with_port)) 

477 self.assertNotEqual(session, store.get(ResourcePath(another_url))) 

478 

479 # Ensure the sessions retrieved from a single store with the same 

480 # root URIs (including port numbers) are equal 

481 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"): 

482 self.assertEqual(session, store.get(ResourcePath(u))) 

483 

484 

485if __name__ == "__main__": 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true

486 unittest.main()