Coverage for tests/test_http.py: 17%
283 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-04 02:38 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-04 02:38 -0800
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import importlib
13import io
14import os.path
15import stat
16import tempfile
17import unittest
18from typing import cast
20import lsst.resources
21import requests
22import responses
23from lsst.resources import ResourcePath
24from lsst.resources._resourceHandles._httpResourceHandle import HttpReadResourceHandle
25from lsst.resources.http import BearerTokenAuth, SessionStore, _is_protected, _is_webdav_endpoint
26from lsst.resources.tests import GenericTestCase
27from lsst.resources.utils import makeTestTempDir, removeTestTempDir
29TESTDIR = os.path.abspath(os.path.dirname(__file__))
32class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
33 scheme = "http"
34 netloc = "server.example"
37class HttpReadWriteTestCase(unittest.TestCase):
38 """Specialist test cases for WebDAV server.
40 The responses class requires that every possible request be explicitly
41 mocked out. This currently makes it extremely inconvenient to subclass
42 the generic read/write tests shared by other URI schemes. For now use
43 explicit standalone tests.
44 """
46 def setUp(self):
47 # Local test directory
48 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
50 serverRoot = "www.not-exists.orgx"
51 existingFolderName = "existingFolder"
52 existingFileName = "existingFile"
53 notExistingFileName = "notExistingFile"
55 fileForHandleWithRange = "handleWithRange"
56 fileForHandleWithOutRange = "handleWithOutRange"
58 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True)
59 self.existingFileResourcePath = ResourcePath(
60 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
61 )
62 self.notExistingFileResourcePath = ResourcePath(
63 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
64 )
65 self.existingFolderResourcePath = ResourcePath(
66 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
67 )
68 self.notExistingFolderResourcePath = ResourcePath(
69 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
70 )
71 self.handleWithRangeResourcePath = ResourcePath(
72 f"https://{serverRoot}/{existingFolderName}/{fileForHandleWithRange}"
73 )
74 self.handleWithOutRangeResourcePath = ResourcePath(
75 f"https://{serverRoot}/{existingFolderName}/{fileForHandleWithOutRange}"
76 )
78 # Used by the handle tests
79 responses.add(
80 responses.HEAD,
81 self.handleWithRangeResourcePath.geturl(),
82 status=200,
83 headers={"Content-Length": "1024", "Accept-Ranges": "true"},
84 )
85 self.handleWithRangeBody = "These are some \n bytes to read"
86 responses.add(
87 responses.GET,
88 self.handleWithRangeResourcePath.geturl(),
89 status=206,
90 body=self.handleWithRangeBody.encode(),
91 )
92 responses.add(
93 responses.PUT,
94 self.handleWithRangeResourcePath.geturl(),
95 status=201,
96 )
98 responses.add(
99 responses.HEAD,
100 self.handleWithOutRangeResourcePath.geturl(),
101 status=200,
102 headers={"Content-Length": "1024"},
103 )
104 responses.add(
105 responses.GET,
106 self.handleWithOutRangeResourcePath.geturl(),
107 status=200,
108 body="These are some bytes to read".encode(),
109 )
111 # Need to declare the options
112 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"})
114 # Used by HttpResourcePath.exists()
115 responses.add(
116 responses.HEAD,
117 self.existingFileResourcePath.geturl(),
118 status=200,
119 headers={"Content-Length": "1024"},
120 )
121 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404)
123 # Used by HttpResourcePath.read()
124 responses.add(
125 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!")
126 )
127 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404)
129 # Used by HttpResourcePath.write()
130 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201)
132 # Used by HttpResourcePath.transfer_from()
133 responses.add(
134 responses.Response(
135 url=self.existingFileResourcePath.geturl(),
136 method="COPY",
137 headers={"Destination": self.existingFileResourcePath.geturl()},
138 status=201,
139 )
140 )
141 responses.add(
142 responses.Response(
143 url=self.existingFileResourcePath.geturl(),
144 method="COPY",
145 headers={"Destination": self.notExistingFileResourcePath.geturl()},
146 status=201,
147 )
148 )
149 responses.add(
150 responses.Response(
151 url=self.existingFileResourcePath.geturl(),
152 method="MOVE",
153 headers={"Destination": self.notExistingFileResourcePath.geturl()},
154 status=201,
155 )
156 )
158 # Used by HttpResourcePath.remove()
159 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200)
160 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404)
162 # Used by HttpResourcePath.mkdir()
163 responses.add(
164 responses.HEAD,
165 self.existingFolderResourcePath.geturl(),
166 status=200,
167 headers={"Content-Length": "1024"},
168 )
169 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"})
170 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404)
171 responses.add(
172 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201)
173 )
174 responses.add(
175 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403)
176 )
178 # Used by HttpResourcePath._do_put()
179 self.redirectPathNoExpect = ResourcePath(f"https://{serverRoot}/redirect-no-expect/file")
180 self.redirectPathExpect = ResourcePath(f"https://{serverRoot}/redirect-expect/file")
181 redirected_url = f"https://{serverRoot}/redirect/location"
182 responses.add(
183 responses.PUT,
184 self.redirectPathNoExpect.geturl(),
185 headers={"Location": redirected_url},
186 status=307,
187 )
188 responses.add(
189 responses.PUT,
190 self.redirectPathExpect.geturl(),
191 headers={"Location": redirected_url},
192 status=307,
193 match=[responses.matchers.header_matcher({"Content-Length": "0", "Expect": "100-continue"})],
194 )
195 responses.add(responses.PUT, redirected_url, status=202)
197 def tearDown(self):
198 if self.tmpdir:
199 if self.tmpdir.isLocal:
200 removeTestTempDir(self.tmpdir.ospath)
202 @responses.activate
203 def test_file_handle(self):
204 # Test that without the correct header the default method is used.
205 with self.handleWithOutRangeResourcePath.open("rb") as handle:
206 self.assertIsInstance(handle, io.BytesIO)
208 # Test that with correct header the correct handle is returned.
209 with self.handleWithRangeResourcePath.open("rb") as handle:
210 self.assertIsInstance(handle, HttpReadResourceHandle)
212 # Test reading byte ranges works
213 with self.handleWithRangeResourcePath.open("rb") as handle:
214 handle = cast(HttpReadResourceHandle, handle)
215 # This is not a real test, because responses can not actually
216 # handle reading sub byte ranges, so the whole thing needs to be
217 # read.
218 result = handle.read(len(self.handleWithRangeBody)).decode()
219 self.assertEqual(result, self.handleWithRangeBody)
220 # Verify there is no internal buffer.
221 self.assertIsNone(handle._completeBuffer)
222 # Verify the position.
223 self.assertEqual(handle.tell(), len(self.handleWithRangeBody))
225 # Jump back to the beginning and test if reading the whole file
226 # prompts the internal buffer to be read.
227 handle.seek(0)
228 self.assertEqual(handle.tell(), 0)
229 result = handle.read().decode()
230 self.assertIsNotNone(handle._completeBuffer)
231 self.assertEqual(result, self.handleWithRangeBody)
233 # Verify reading as a string handle works as expected.
234 with self.handleWithRangeResourcePath.open("r") as handle:
235 self.assertIsInstance(handle, io.TextIOWrapper)
237 handle = cast(io.TextIOWrapper, handle)
238 self.assertIsInstance(handle.buffer, HttpReadResourceHandle)
240 # Check if string methods work.
241 result = handle.read()
242 self.assertEqual(result, self.handleWithRangeBody)
244 # Verify that write modes invoke the default base method
245 with self.handleWithRangeResourcePath.open("w") as handle:
246 self.assertIsInstance(handle, io.StringIO)
248 @responses.activate
249 def test_exists(self):
250 self.assertTrue(self.existingFileResourcePath.exists())
251 self.assertFalse(self.notExistingFileResourcePath.exists())
253 self.assertEqual(self.existingFileResourcePath.size(), 1024)
254 with self.assertRaises(FileNotFoundError):
255 self.notExistingFileResourcePath.size()
257 @responses.activate
258 def test_remove(self):
259 self.assertIsNone(self.existingFileResourcePath.remove())
260 with self.assertRaises(FileNotFoundError):
261 self.notExistingFileResourcePath.remove()
263 url = "https://example.org/delete"
264 responses.add(responses.DELETE, url, status=404)
265 with self.assertRaises(FileNotFoundError):
266 ResourcePath(url).remove()
268 @responses.activate
269 def test_mkdir(self):
270 # The mock means that we can't check this now exists
271 self.notExistingFolderResourcePath.mkdir()
273 # This should do nothing
274 self.existingFolderResourcePath.mkdir()
276 with self.assertRaises(ValueError):
277 self.notExistingFileResourcePath.mkdir()
279 @responses.activate
280 def test_read(self):
281 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!")
282 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.")
283 with self.assertRaises(FileNotFoundError):
284 self.notExistingFileResourcePath.read()
286 # Run this twice to ensure use of cache in code coverage.
287 for _ in (1, 2):
288 with self.existingFileResourcePath.as_local() as local_uri:
289 self.assertTrue(local_uri.isLocal)
290 content = local_uri.read().decode()
291 self.assertEqual(content, "It works!")
293 # Check that the environment variable is being read.
294 lsst.resources.http._TMPDIR = None
295 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir.ospath}):
296 with self.existingFileResourcePath.as_local() as local_uri:
297 self.assertTrue(local_uri.isLocal)
298 content = local_uri.read().decode()
299 self.assertEqual(content, "It works!")
300 self.assertIsNotNone(local_uri.relative_to(self.tmpdir))
302 @responses.activate
303 def test_write(self):
304 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content.")))
305 with self.assertRaises(FileExistsError):
306 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False)
308 url = "https://example.org/put"
309 responses.add(responses.PUT, url, status=404)
310 with self.assertRaises(ValueError):
311 ResourcePath(url).write(data=str.encode("Some content."))
313 @responses.activate
314 def test_do_put_with_redirection(self):
315 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER.
316 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
317 importlib.reload(lsst.resources.http)
318 body = str.encode("any contents")
319 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body))
321 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER.
322 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True):
323 importlib.reload(lsst.resources.http)
324 self.assertIsNone(self.redirectPathExpect._do_put(data=body))
326 @responses.activate
327 def test_transfer(self):
328 # Transferring to self should be no-op.
329 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
331 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath))
332 # Should test for existence.
333 # self.assertTrue(self.notExistingFileResourcePath.exists())
335 # Should delete and try again with move.
336 # self.notExistingFileResourcePath.remove()
337 self.assertIsNone(
338 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move")
339 )
340 # Should then check that it was moved.
341 # self.assertFalse(self.existingFileResourcePath.exists())
343 # Existing file resource should have been removed so this should
344 # trigger FileNotFoundError.
345 # with self.assertRaises(FileNotFoundError):
346 # self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
347 with self.assertRaises(ValueError):
348 self.notExistingFileResourcePath.transfer_from(
349 src=self.existingFileResourcePath, transfer="unsupported"
350 )
352 def test_parent(self):
353 self.assertEqual(
354 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl()
355 )
356 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
357 self.assertEqual(
358 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl()
359 )
361 def test_send_expect_header(self):
362 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from
363 # the environment.
364 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
365 importlib.reload(lsst.resources.http)
366 self.assertFalse(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
368 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True):
369 importlib.reload(lsst.resources.http)
370 self.assertTrue(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
372 def test_timeout(self):
373 connect_timeout = 100
374 read_timeout = 200
375 with unittest.mock.patch.dict(
376 os.environ,
377 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)},
378 clear=True,
379 ):
380 # Force module reload to initialize TIMEOUT.
381 importlib.reload(lsst.resources.http)
382 self.assertEqual(lsst.resources.http.TIMEOUT, (connect_timeout, read_timeout))
384 def test_is_protected(self):
385 self.assertFalse(_is_protected("/this-file-does-not-exist"))
387 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
388 f.write("XXXX")
389 file_path = f.name
391 os.chmod(file_path, stat.S_IRUSR)
392 self.assertTrue(_is_protected(file_path))
394 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
395 os.chmod(file_path, stat.S_IRUSR | mode)
396 self.assertFalse(_is_protected(file_path))
399class WebdavUtilsTestCase(unittest.TestCase):
400 """Test for the Webdav related utilities."""
402 serverRoot = "www.lsstwithwebdav.orgx"
403 wrongRoot = "www.lsstwithoutwebdav.org"
405 def setUp(self):
406 responses.add(responses.OPTIONS, f"https://{self.serverRoot}", status=200, headers={"DAV": "1,2,3"})
407 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200)
409 @responses.activate
410 def test_is_webdav_endpoint(self):
411 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}"))
412 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}"))
415class BearerTokenAuthTestCase(unittest.TestCase):
416 """Test for the BearerTokenAuth class."""
418 def setUp(self):
419 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
420 self.token = "ABCDE1234"
422 def tearDown(self):
423 if self.tmpdir and self.tmpdir.isLocal:
424 removeTestTempDir(self.tmpdir.ospath)
426 def test_empty_token(self):
427 """Ensure that when no token is provided the request is not
428 modified.
429 """
430 auth = BearerTokenAuth(None)
431 auth._refresh()
432 self.assertIsNone(auth._token)
433 self.assertIsNone(auth._path)
434 req = requests.Request("GET", "https://example.org")
435 self.assertEqual(auth(req), req)
437 def test_token_value(self):
438 """Ensure that when a token value is provided, the 'Authorization'
439 header is added to the requests.
440 """
441 auth = BearerTokenAuth(self.token)
442 req = auth(requests.Request("GET", "https://example.org").prepare())
443 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
445 def test_token_file(self):
446 """Ensure when the provided token is a file path, its contents is
447 correctly used in the the 'Authorization' header of the requests.
448 """
449 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
450 f.write(self.token)
451 token_file_path = f.name
453 # Ensure the request's "Authorization" header is set with the right
454 # token value
455 os.chmod(token_file_path, stat.S_IRUSR)
456 auth = BearerTokenAuth(token_file_path)
457 req = auth(requests.Request("GET", "https://example.org").prepare())
458 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
460 # Ensure an exception is raised if either group or other can read the
461 # token file
462 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
463 os.chmod(token_file_path, stat.S_IRUSR | mode)
464 with self.assertRaises(PermissionError):
465 BearerTokenAuth(token_file_path)
468class SessionStoreTestCase(unittest.TestCase):
469 """Test for the SessionStore class."""
471 def setUp(self):
472 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
473 self.rpath = ResourcePath("https://example.org")
475 def tearDown(self):
476 if self.tmpdir and self.tmpdir.isLocal:
477 removeTestTempDir(self.tmpdir.ospath)
479 def test_ca_cert_bundle(self):
480 """Ensure a certificate authorities bundle is used to authentify
481 the remote server.
482 """
483 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
484 f.write("CERT BUNDLE")
485 cert_bundle = f.name
487 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True):
488 session = SessionStore().get(self.rpath)
489 self.assertEqual(session.verify, cert_bundle)
491 def test_user_cert(self):
492 """Ensure if user certificate and private key are provided, they are
493 used for authenticating the client.
494 """
496 # Create mock certificate and private key files.
497 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
498 f.write("CERT")
499 client_cert = f.name
501 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
502 f.write("KEY")
503 client_key = f.name
505 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY
506 # must be initialized.
507 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True):
508 with self.assertRaises(ValueError):
509 SessionStore().get(self.rpath)
511 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True):
512 with self.assertRaises(ValueError):
513 SessionStore().get(self.rpath)
515 # Check private key file must be accessible only by its owner.
516 with unittest.mock.patch.dict(
517 os.environ,
518 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key},
519 clear=True,
520 ):
521 # Ensure the session client certificate is initialized when
522 # only the owner can read the private key file.
523 os.chmod(client_key, stat.S_IRUSR)
524 session = SessionStore().get(self.rpath)
525 self.assertEqual(session.cert[0], client_cert)
526 self.assertEqual(session.cert[1], client_key)
528 # Ensure an exception is raised if either group or other can access
529 # the private key file.
530 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
531 os.chmod(client_key, stat.S_IRUSR | mode)
532 with self.assertRaises(PermissionError):
533 SessionStore().get(self.rpath)
535 def test_token_env(self):
536 """Ensure when the token is provided via an environment variable
537 the sessions are equipped with a BearerTokenAuth.
538 """
539 token = "ABCDE"
540 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True):
541 session = SessionStore().get(self.rpath)
542 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth)
543 self.assertEqual(session.auth._token, token)
544 self.assertIsNone(session.auth._path)
546 def test_sessions(self):
547 """Ensure the session caching mechanism works."""
549 # Ensure the store provides a session for a given URL
550 root_url = "https://example.org"
551 store = SessionStore()
552 session = store.get(ResourcePath(root_url))
553 self.assertIsNotNone(session)
555 # Ensure the sessions retrieved from a single store with the same
556 # root URIs are equal
557 for u in (f"{root_url}", f"{root_url}/path/to/file"):
558 self.assertEqual(session, store.get(ResourcePath(u)))
560 # Ensure sessions retrieved for different root URIs are different
561 another_url = "https://another.example.org"
562 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
564 # Ensure the sessions retrieved from a single store for URLs with
565 # different port numbers are different
566 root_url_with_port = f"{another_url}:12345"
567 session = store.get(ResourcePath(root_url_with_port))
568 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
570 # Ensure the sessions retrieved from a single store with the same
571 # root URIs (including port numbers) are equal
572 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"):
573 self.assertEqual(session, store.get(ResourcePath(u)))
576if __name__ == "__main__": 576 ↛ 577line 576 didn't jump to line 577, because the condition on line 576 was never true
577 unittest.main()