Coverage for tests/test_http.py: 20%
245 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:04 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-09-11 01:04 -0700
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import importlib
13import os.path
14import stat
15import tempfile
16import unittest
18import lsst.resources
19import requests
20import responses
21from lsst.resources import ResourcePath
22from lsst.resources.http import BearerTokenAuth, SessionStore, _is_protected, _is_webdav_endpoint
23from lsst.resources.tests import GenericTestCase
24from lsst.resources.utils import makeTestTempDir, removeTestTempDir
26TESTDIR = os.path.abspath(os.path.dirname(__file__))
29class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
30 scheme = "http"
31 netloc = "server.example"
34class HttpReadWriteTestCase(unittest.TestCase):
35 """Specialist test cases for WebDAV server.
37 The responses class requires that every possible request be explicitly
38 mocked out. This currently makes it extremely inconvenient to subclass
39 the generic read/write tests shared by other URI schemes. For now use
40 explicit standalone tests.
41 """
43 def setUp(self):
44 # Local test directory
45 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
47 serverRoot = "www.not-exists.orgx"
48 existingFolderName = "existingFolder"
49 existingFileName = "existingFile"
50 notExistingFileName = "notExistingFile"
52 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True)
53 self.existingFileResourcePath = ResourcePath(
54 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
55 )
56 self.notExistingFileResourcePath = ResourcePath(
57 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
58 )
59 self.existingFolderResourcePath = ResourcePath(
60 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
61 )
62 self.notExistingFolderResourcePath = ResourcePath(
63 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
64 )
66 # Need to declare the options
67 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"})
69 # Used by HttpResourcePath.exists()
70 responses.add(
71 responses.HEAD,
72 self.existingFileResourcePath.geturl(),
73 status=200,
74 headers={"Content-Length": "1024"},
75 )
76 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404)
78 # Used by HttpResourcePath.read()
79 responses.add(
80 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!")
81 )
82 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404)
84 # Used by HttpResourcePath.write()
85 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201)
87 # Used by HttpResourcePath.transfer_from()
88 responses.add(
89 responses.Response(
90 url=self.existingFileResourcePath.geturl(),
91 method="COPY",
92 headers={"Destination": self.existingFileResourcePath.geturl()},
93 status=201,
94 )
95 )
96 responses.add(
97 responses.Response(
98 url=self.existingFileResourcePath.geturl(),
99 method="COPY",
100 headers={"Destination": self.notExistingFileResourcePath.geturl()},
101 status=201,
102 )
103 )
104 responses.add(
105 responses.Response(
106 url=self.existingFileResourcePath.geturl(),
107 method="MOVE",
108 headers={"Destination": self.notExistingFileResourcePath.geturl()},
109 status=201,
110 )
111 )
113 # Used by HttpResourcePath.remove()
114 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200)
115 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404)
117 # Used by HttpResourcePath.mkdir()
118 responses.add(
119 responses.HEAD,
120 self.existingFolderResourcePath.geturl(),
121 status=200,
122 headers={"Content-Length": "1024"},
123 )
124 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"})
125 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404)
126 responses.add(
127 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201)
128 )
129 responses.add(
130 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403)
131 )
133 # Used by HttpResourcePath._do_put()
134 self.redirectPathNoExpect = ResourcePath(f"https://{serverRoot}/redirect-no-expect/file")
135 self.redirectPathExpect = ResourcePath(f"https://{serverRoot}/redirect-expect/file")
136 redirected_url = f"https://{serverRoot}/redirect/location"
137 responses.add(
138 responses.PUT,
139 self.redirectPathNoExpect.geturl(),
140 headers={"Location": redirected_url},
141 status=307,
142 )
143 responses.add(
144 responses.PUT,
145 self.redirectPathExpect.geturl(),
146 headers={"Location": redirected_url},
147 status=307,
148 match=[responses.matchers.header_matcher({"Content-Length": "0", "Expect": "100-continue"})],
149 )
150 responses.add(responses.PUT, redirected_url, status=202)
152 def tearDown(self):
153 if self.tmpdir:
154 if self.tmpdir.isLocal:
155 removeTestTempDir(self.tmpdir.ospath)
157 @responses.activate
158 def test_exists(self):
160 self.assertTrue(self.existingFileResourcePath.exists())
161 self.assertFalse(self.notExistingFileResourcePath.exists())
163 self.assertEqual(self.existingFileResourcePath.size(), 1024)
164 with self.assertRaises(FileNotFoundError):
165 self.notExistingFileResourcePath.size()
167 @responses.activate
168 def test_remove(self):
170 self.assertIsNone(self.existingFileResourcePath.remove())
171 with self.assertRaises(FileNotFoundError):
172 self.notExistingFileResourcePath.remove()
174 url = "https://example.org/delete"
175 responses.add(responses.DELETE, url, status=404)
176 with self.assertRaises(FileNotFoundError):
177 ResourcePath(url).remove()
179 @responses.activate
180 def test_mkdir(self):
182 # The mock means that we can't check this now exists
183 self.notExistingFolderResourcePath.mkdir()
185 # This should do nothing
186 self.existingFolderResourcePath.mkdir()
188 with self.assertRaises(ValueError):
189 self.notExistingFileResourcePath.mkdir()
191 @responses.activate
192 def test_read(self):
194 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!")
195 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.")
196 with self.assertRaises(FileNotFoundError):
197 self.notExistingFileResourcePath.read()
199 # Run this twice to ensure use of cache in code coverage.
200 for _ in (1, 2):
201 with self.existingFileResourcePath.as_local() as local_uri:
202 self.assertTrue(local_uri.isLocal)
203 content = local_uri.read().decode()
204 self.assertEqual(content, "It works!")
206 # Check that the environment variable is being read.
207 lsst.resources.http._TMPDIR = None
208 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir.ospath}):
209 with self.existingFileResourcePath.as_local() as local_uri:
210 self.assertTrue(local_uri.isLocal)
211 content = local_uri.read().decode()
212 self.assertEqual(content, "It works!")
213 self.assertIsNotNone(local_uri.relative_to(self.tmpdir))
215 @responses.activate
216 def test_write(self):
218 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content.")))
219 with self.assertRaises(FileExistsError):
220 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False)
222 url = "https://example.org/put"
223 responses.add(responses.PUT, url, status=404)
224 with self.assertRaises(ValueError):
225 ResourcePath(url).write(data=str.encode("Some content."))
227 @responses.activate
228 def test_do_put_with_redirection(self):
230 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER.
231 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
232 importlib.reload(lsst.resources.http)
233 body = str.encode("any contents")
234 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body))
236 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER.
237 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True):
238 importlib.reload(lsst.resources.http)
239 self.assertIsNone(self.redirectPathExpect._do_put(data=body))
241 @responses.activate
242 def test_transfer(self):
244 # Transferring to self should be no-op.
245 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
247 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath))
248 # Should test for existence.
249 # self.assertTrue(self.notExistingFileResourcePath.exists())
251 # Should delete and try again with move.
252 # self.notExistingFileResourcePath.remove()
253 self.assertIsNone(
254 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move")
255 )
256 # Should then check that it was moved.
257 # self.assertFalse(self.existingFileResourcePath.exists())
259 # Existing file resource should have been removed so this should
260 # trigger FileNotFoundError.
261 # with self.assertRaises(FileNotFoundError):
262 # self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
263 with self.assertRaises(ValueError):
264 self.notExistingFileResourcePath.transfer_from(
265 src=self.existingFileResourcePath, transfer="unsupported"
266 )
268 def test_parent(self):
270 self.assertEqual(
271 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl()
272 )
273 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
274 self.assertEqual(
275 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl()
276 )
278 def test_send_expect_header(self):
280 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from
281 # the environment.
282 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
283 importlib.reload(lsst.resources.http)
284 self.assertFalse(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
286 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True):
287 importlib.reload(lsst.resources.http)
288 self.assertTrue(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
290 def test_timeout(self):
292 connect_timeout = 100
293 read_timeout = 200
294 with unittest.mock.patch.dict(
295 os.environ,
296 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)},
297 clear=True,
298 ):
299 # Force module reload to initialize TIMEOUT.
300 importlib.reload(lsst.resources.http)
301 self.assertEqual(lsst.resources.http.TIMEOUT, (connect_timeout, read_timeout))
303 def test_is_protected(self):
305 self.assertFalse(_is_protected("/this-file-does-not-exist"))
307 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
308 f.write("XXXX")
309 file_path = f.name
311 os.chmod(file_path, stat.S_IRUSR)
312 self.assertTrue(_is_protected(file_path))
314 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
315 os.chmod(file_path, stat.S_IRUSR | mode)
316 self.assertFalse(_is_protected(file_path))
319class WebdavUtilsTestCase(unittest.TestCase):
320 """Test for the Webdav related utilities."""
322 serverRoot = "www.lsstwithwebdav.orgx"
323 wrongRoot = "www.lsstwithoutwebdav.org"
325 def setUp(self):
326 responses.add(responses.OPTIONS, f"https://{self.serverRoot}", status=200, headers={"DAV": "1,2,3"})
327 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200)
329 @responses.activate
330 def test_is_webdav_endpoint(self):
332 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}"))
333 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}"))
336class BearerTokenAuthTestCase(unittest.TestCase):
337 """Test for the BearerTokenAuth class."""
339 def setUp(self):
340 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
341 self.token = "ABCDE1234"
343 def tearDown(self):
344 if self.tmpdir and self.tmpdir.isLocal:
345 removeTestTempDir(self.tmpdir.ospath)
347 def test_empty_token(self):
348 """Ensure that when no token is provided the request is not
349 modified.
350 """
351 auth = BearerTokenAuth(None)
352 auth._refresh()
353 self.assertIsNone(auth._token)
354 self.assertIsNone(auth._path)
355 req = requests.Request("GET", "https://example.org")
356 self.assertEqual(auth(req), req)
358 def test_token_value(self):
359 """Ensure that when a token value is provided, the 'Authorization'
360 header is added to the requests.
361 """
362 auth = BearerTokenAuth(self.token)
363 req = auth(requests.Request("GET", "https://example.org").prepare())
364 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
366 def test_token_file(self):
367 """Ensure when the provided token is a file path, its contents is
368 correctly used in the the 'Authorization' header of the requests.
369 """
370 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
371 f.write(self.token)
372 token_file_path = f.name
374 # Ensure the request's "Authorization" header is set with the right
375 # token value
376 os.chmod(token_file_path, stat.S_IRUSR)
377 auth = BearerTokenAuth(token_file_path)
378 req = auth(requests.Request("GET", "https://example.org").prepare())
379 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
381 # Ensure an exception is raised if either group or other can read the
382 # token file
383 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
384 os.chmod(token_file_path, stat.S_IRUSR | mode)
385 with self.assertRaises(PermissionError):
386 BearerTokenAuth(token_file_path)
389class SessionStoreTestCase(unittest.TestCase):
390 """Test for the SessionStore class."""
392 def setUp(self):
393 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
394 self.rpath = ResourcePath("https://example.org")
396 def tearDown(self):
397 if self.tmpdir and self.tmpdir.isLocal:
398 removeTestTempDir(self.tmpdir.ospath)
400 def test_ca_cert_bundle(self):
401 """Ensure a certificate authorities bundle is used to authentify
402 the remote server.
403 """
404 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
405 f.write("CERT BUNDLE")
406 cert_bundle = f.name
408 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True):
409 session = SessionStore().get(self.rpath)
410 self.assertEqual(session.verify, cert_bundle)
412 def test_user_cert(self):
413 """Ensure if user certificate and private key are provided, they are
414 used for authenticating the client.
415 """
417 # Create mock certificate and private key files.
418 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
419 f.write("CERT")
420 client_cert = f.name
422 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
423 f.write("KEY")
424 client_key = f.name
426 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY
427 # must be initialized.
428 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True):
429 with self.assertRaises(ValueError):
430 SessionStore().get(self.rpath)
432 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True):
433 with self.assertRaises(ValueError):
434 SessionStore().get(self.rpath)
436 # Check private key file must be accessible only by its owner.
437 with unittest.mock.patch.dict(
438 os.environ,
439 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key},
440 clear=True,
441 ):
442 # Ensure the session client certificate is initialized when
443 # only the owner can read the private key file.
444 os.chmod(client_key, stat.S_IRUSR)
445 session = SessionStore().get(self.rpath)
446 self.assertEqual(session.cert[0], client_cert)
447 self.assertEqual(session.cert[1], client_key)
449 # Ensure an exception is raised if either group or other can access
450 # the private key file.
451 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
452 os.chmod(client_key, stat.S_IRUSR | mode)
453 with self.assertRaises(PermissionError):
454 SessionStore().get(self.rpath)
456 def test_token_env(self):
457 """Ensure when the token is provided via an environment variable
458 the sessions are equipped with a BearerTokenAuth.
459 """
460 token = "ABCDE"
461 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True):
462 session = SessionStore().get(self.rpath)
463 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth)
464 self.assertEqual(session.auth._token, token)
465 self.assertIsNone(session.auth._path)
467 def test_sessions(self):
468 """Ensure the session caching mechanism works."""
470 # Ensure the store provides a session for a given URL
471 root_url = "https://example.org"
472 store = SessionStore()
473 session = store.get(ResourcePath(root_url))
474 self.assertIsNotNone(session)
476 # Ensure the sessions retrieved from a single store with the same
477 # root URIs are equal
478 for u in (f"{root_url}", f"{root_url}/path/to/file"):
479 self.assertEqual(session, store.get(ResourcePath(u)))
481 # Ensure sessions retrieved for different root URIs are different
482 another_url = "https://another.example.org"
483 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
485 # Ensure the sessions retrieved from a single store for URLs with
486 # different port numbers are different
487 root_url_with_port = f"{another_url}:12345"
488 session = store.get(ResourcePath(root_url_with_port))
489 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
491 # Ensure the sessions retrieved from a single store with the same
492 # root URIs (including port numbers) are equal
493 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"):
494 self.assertEqual(session, store.get(ResourcePath(u)))
497if __name__ == "__main__": 497 ↛ 498line 497 didn't jump to line 498, because the condition on line 497 was never true
498 unittest.main()