Coverage for tests/test_http.py: 18%
245 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 00:05 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-15 00:05 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import importlib
13import os.path
14import stat
15import tempfile
16import unittest
18import lsst.resources
19import requests
20import responses
21from lsst.resources import ResourcePath
22from lsst.resources.http import BearerTokenAuth, SessionStore, _is_protected, _is_webdav_endpoint
23from lsst.resources.tests import GenericTestCase
24from lsst.resources.utils import makeTestTempDir, removeTestTempDir
26TESTDIR = os.path.abspath(os.path.dirname(__file__))
29class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
30 scheme = "http"
31 netloc = "server.example"
34class HttpReadWriteTestCase(unittest.TestCase):
35 """Specialist test cases for WebDAV server.
37 The responses class requires that every possible request be explicitly
38 mocked out. This currently makes it extremely inconvenient to subclass
39 the generic read/write tests shared by other URI schemes. For now use
40 explicit standalone tests.
41 """
43 def setUp(self):
44 # Local test directory
45 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
47 serverRoot = "www.not-exists.orgx"
48 existingFolderName = "existingFolder"
49 existingFileName = "existingFile"
50 notExistingFileName = "notExistingFile"
52 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True)
53 self.existingFileResourcePath = ResourcePath(
54 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
55 )
56 self.notExistingFileResourcePath = ResourcePath(
57 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
58 )
59 self.existingFolderResourcePath = ResourcePath(
60 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
61 )
62 self.notExistingFolderResourcePath = ResourcePath(
63 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
64 )
66 # Need to declare the options
67 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"})
69 # Used by HttpResourcePath.exists()
70 responses.add(
71 responses.HEAD,
72 self.existingFileResourcePath.geturl(),
73 status=200,
74 headers={"Content-Length": "1024"},
75 )
76 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404)
78 # Used by HttpResourcePath.read()
79 responses.add(
80 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!")
81 )
82 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404)
84 # Used by HttpResourcePath.write()
85 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201)
87 # Used by HttpResourcePath.transfer_from()
88 responses.add(
89 responses.Response(
90 url=self.existingFileResourcePath.geturl(),
91 method="COPY",
92 headers={"Destination": self.existingFileResourcePath.geturl()},
93 status=201,
94 )
95 )
96 responses.add(
97 responses.Response(
98 url=self.existingFileResourcePath.geturl(),
99 method="COPY",
100 headers={"Destination": self.notExistingFileResourcePath.geturl()},
101 status=201,
102 )
103 )
104 responses.add(
105 responses.Response(
106 url=self.existingFileResourcePath.geturl(),
107 method="MOVE",
108 headers={"Destination": self.notExistingFileResourcePath.geturl()},
109 status=201,
110 )
111 )
113 # Used by HttpResourcePath.remove()
114 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200)
115 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404)
117 # Used by HttpResourcePath.mkdir()
118 responses.add(
119 responses.HEAD,
120 self.existingFolderResourcePath.geturl(),
121 status=200,
122 headers={"Content-Length": "1024"},
123 )
124 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"})
125 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404)
126 responses.add(
127 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201)
128 )
129 responses.add(
130 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403)
131 )
133 # Used by HttpResourcePath._do_put()
134 self.redirectPathNoExpect = ResourcePath(f"https://{serverRoot}/redirect-no-expect/file")
135 self.redirectPathExpect = ResourcePath(f"https://{serverRoot}/redirect-expect/file")
136 redirected_url = f"https://{serverRoot}/redirect/location"
137 responses.add(
138 responses.PUT,
139 self.redirectPathNoExpect.geturl(),
140 headers={"Location": redirected_url},
141 status=307,
142 )
143 responses.add(
144 responses.PUT,
145 self.redirectPathExpect.geturl(),
146 headers={"Location": redirected_url},
147 status=307,
148 match=[responses.matchers.header_matcher({"Content-Length": "0", "Expect": "100-continue"})],
149 )
150 responses.add(responses.PUT, redirected_url, status=202)
152 def tearDown(self):
153 if self.tmpdir:
154 if self.tmpdir.isLocal:
155 removeTestTempDir(self.tmpdir.ospath)
157 @responses.activate
158 def test_exists(self):
159 self.assertTrue(self.existingFileResourcePath.exists())
160 self.assertFalse(self.notExistingFileResourcePath.exists())
162 self.assertEqual(self.existingFileResourcePath.size(), 1024)
163 with self.assertRaises(FileNotFoundError):
164 self.notExistingFileResourcePath.size()
166 @responses.activate
167 def test_remove(self):
168 self.assertIsNone(self.existingFileResourcePath.remove())
169 with self.assertRaises(FileNotFoundError):
170 self.notExistingFileResourcePath.remove()
172 url = "https://example.org/delete"
173 responses.add(responses.DELETE, url, status=404)
174 with self.assertRaises(FileNotFoundError):
175 ResourcePath(url).remove()
177 @responses.activate
178 def test_mkdir(self):
179 # The mock means that we can't check this now exists
180 self.notExistingFolderResourcePath.mkdir()
182 # This should do nothing
183 self.existingFolderResourcePath.mkdir()
185 with self.assertRaises(ValueError):
186 self.notExistingFileResourcePath.mkdir()
188 @responses.activate
189 def test_read(self):
190 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!")
191 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.")
192 with self.assertRaises(FileNotFoundError):
193 self.notExistingFileResourcePath.read()
195 # Run this twice to ensure use of cache in code coverage.
196 for _ in (1, 2):
197 with self.existingFileResourcePath.as_local() as local_uri:
198 self.assertTrue(local_uri.isLocal)
199 content = local_uri.read().decode()
200 self.assertEqual(content, "It works!")
202 # Check that the environment variable is being read.
203 lsst.resources.http._TMPDIR = None
204 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir.ospath}):
205 with self.existingFileResourcePath.as_local() as local_uri:
206 self.assertTrue(local_uri.isLocal)
207 content = local_uri.read().decode()
208 self.assertEqual(content, "It works!")
209 self.assertIsNotNone(local_uri.relative_to(self.tmpdir))
211 @responses.activate
212 def test_write(self):
213 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content.")))
214 with self.assertRaises(FileExistsError):
215 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False)
217 url = "https://example.org/put"
218 responses.add(responses.PUT, url, status=404)
219 with self.assertRaises(ValueError):
220 ResourcePath(url).write(data=str.encode("Some content."))
222 @responses.activate
223 def test_do_put_with_redirection(self):
224 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER.
225 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
226 importlib.reload(lsst.resources.http)
227 body = str.encode("any contents")
228 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body))
230 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER.
231 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True):
232 importlib.reload(lsst.resources.http)
233 self.assertIsNone(self.redirectPathExpect._do_put(data=body))
235 @responses.activate
236 def test_transfer(self):
237 # Transferring to self should be no-op.
238 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
240 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath))
241 # Should test for existence.
242 # self.assertTrue(self.notExistingFileResourcePath.exists())
244 # Should delete and try again with move.
245 # self.notExistingFileResourcePath.remove()
246 self.assertIsNone(
247 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move")
248 )
249 # Should then check that it was moved.
250 # self.assertFalse(self.existingFileResourcePath.exists())
252 # Existing file resource should have been removed so this should
253 # trigger FileNotFoundError.
254 # with self.assertRaises(FileNotFoundError):
255 # self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
256 with self.assertRaises(ValueError):
257 self.notExistingFileResourcePath.transfer_from(
258 src=self.existingFileResourcePath, transfer="unsupported"
259 )
261 def test_parent(self):
262 self.assertEqual(
263 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl()
264 )
265 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
266 self.assertEqual(
267 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl()
268 )
270 def test_send_expect_header(self):
271 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from
272 # the environment.
273 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
274 importlib.reload(lsst.resources.http)
275 self.assertFalse(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
277 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True):
278 importlib.reload(lsst.resources.http)
279 self.assertTrue(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
281 def test_timeout(self):
282 connect_timeout = 100
283 read_timeout = 200
284 with unittest.mock.patch.dict(
285 os.environ,
286 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)},
287 clear=True,
288 ):
289 # Force module reload to initialize TIMEOUT.
290 importlib.reload(lsst.resources.http)
291 self.assertEqual(lsst.resources.http.TIMEOUT, (connect_timeout, read_timeout))
293 def test_is_protected(self):
294 self.assertFalse(_is_protected("/this-file-does-not-exist"))
296 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
297 f.write("XXXX")
298 file_path = f.name
300 os.chmod(file_path, stat.S_IRUSR)
301 self.assertTrue(_is_protected(file_path))
303 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
304 os.chmod(file_path, stat.S_IRUSR | mode)
305 self.assertFalse(_is_protected(file_path))
308class WebdavUtilsTestCase(unittest.TestCase):
309 """Test for the Webdav related utilities."""
311 serverRoot = "www.lsstwithwebdav.orgx"
312 wrongRoot = "www.lsstwithoutwebdav.org"
314 def setUp(self):
315 responses.add(responses.OPTIONS, f"https://{self.serverRoot}", status=200, headers={"DAV": "1,2,3"})
316 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200)
318 @responses.activate
319 def test_is_webdav_endpoint(self):
320 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}"))
321 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}"))
324class BearerTokenAuthTestCase(unittest.TestCase):
325 """Test for the BearerTokenAuth class."""
327 def setUp(self):
328 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
329 self.token = "ABCDE1234"
331 def tearDown(self):
332 if self.tmpdir and self.tmpdir.isLocal:
333 removeTestTempDir(self.tmpdir.ospath)
335 def test_empty_token(self):
336 """Ensure that when no token is provided the request is not
337 modified.
338 """
339 auth = BearerTokenAuth(None)
340 auth._refresh()
341 self.assertIsNone(auth._token)
342 self.assertIsNone(auth._path)
343 req = requests.Request("GET", "https://example.org")
344 self.assertEqual(auth(req), req)
346 def test_token_value(self):
347 """Ensure that when a token value is provided, the 'Authorization'
348 header is added to the requests.
349 """
350 auth = BearerTokenAuth(self.token)
351 req = auth(requests.Request("GET", "https://example.org").prepare())
352 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
354 def test_token_file(self):
355 """Ensure when the provided token is a file path, its contents is
356 correctly used in the the 'Authorization' header of the requests.
357 """
358 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
359 f.write(self.token)
360 token_file_path = f.name
362 # Ensure the request's "Authorization" header is set with the right
363 # token value
364 os.chmod(token_file_path, stat.S_IRUSR)
365 auth = BearerTokenAuth(token_file_path)
366 req = auth(requests.Request("GET", "https://example.org").prepare())
367 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
369 # Ensure an exception is raised if either group or other can read the
370 # token file
371 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
372 os.chmod(token_file_path, stat.S_IRUSR | mode)
373 with self.assertRaises(PermissionError):
374 BearerTokenAuth(token_file_path)
377class SessionStoreTestCase(unittest.TestCase):
378 """Test for the SessionStore class."""
380 def setUp(self):
381 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
382 self.rpath = ResourcePath("https://example.org")
384 def tearDown(self):
385 if self.tmpdir and self.tmpdir.isLocal:
386 removeTestTempDir(self.tmpdir.ospath)
388 def test_ca_cert_bundle(self):
389 """Ensure a certificate authorities bundle is used to authentify
390 the remote server.
391 """
392 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
393 f.write("CERT BUNDLE")
394 cert_bundle = f.name
396 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True):
397 session = SessionStore().get(self.rpath)
398 self.assertEqual(session.verify, cert_bundle)
400 def test_user_cert(self):
401 """Ensure if user certificate and private key are provided, they are
402 used for authenticating the client.
403 """
405 # Create mock certificate and private key files.
406 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
407 f.write("CERT")
408 client_cert = f.name
410 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
411 f.write("KEY")
412 client_key = f.name
414 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY
415 # must be initialized.
416 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True):
417 with self.assertRaises(ValueError):
418 SessionStore().get(self.rpath)
420 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True):
421 with self.assertRaises(ValueError):
422 SessionStore().get(self.rpath)
424 # Check private key file must be accessible only by its owner.
425 with unittest.mock.patch.dict(
426 os.environ,
427 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key},
428 clear=True,
429 ):
430 # Ensure the session client certificate is initialized when
431 # only the owner can read the private key file.
432 os.chmod(client_key, stat.S_IRUSR)
433 session = SessionStore().get(self.rpath)
434 self.assertEqual(session.cert[0], client_cert)
435 self.assertEqual(session.cert[1], client_key)
437 # Ensure an exception is raised if either group or other can access
438 # the private key file.
439 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
440 os.chmod(client_key, stat.S_IRUSR | mode)
441 with self.assertRaises(PermissionError):
442 SessionStore().get(self.rpath)
444 def test_token_env(self):
445 """Ensure when the token is provided via an environment variable
446 the sessions are equipped with a BearerTokenAuth.
447 """
448 token = "ABCDE"
449 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True):
450 session = SessionStore().get(self.rpath)
451 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth)
452 self.assertEqual(session.auth._token, token)
453 self.assertIsNone(session.auth._path)
455 def test_sessions(self):
456 """Ensure the session caching mechanism works."""
458 # Ensure the store provides a session for a given URL
459 root_url = "https://example.org"
460 store = SessionStore()
461 session = store.get(ResourcePath(root_url))
462 self.assertIsNotNone(session)
464 # Ensure the sessions retrieved from a single store with the same
465 # root URIs are equal
466 for u in (f"{root_url}", f"{root_url}/path/to/file"):
467 self.assertEqual(session, store.get(ResourcePath(u)))
469 # Ensure sessions retrieved for different root URIs are different
470 another_url = "https://another.example.org"
471 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
473 # Ensure the sessions retrieved from a single store for URLs with
474 # different port numbers are different
475 root_url_with_port = f"{another_url}:12345"
476 session = store.get(ResourcePath(root_url_with_port))
477 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
479 # Ensure the sessions retrieved from a single store with the same
480 # root URIs (including port numbers) are equal
481 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"):
482 self.assertEqual(session, store.get(ResourcePath(u)))
485if __name__ == "__main__": 485 ↛ 486line 485 didn't jump to line 486, because the condition on line 485 was never true
486 unittest.main()