Coverage for tests/test_http.py: 17%
283 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-01 02:02 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-01 02:02 -0800
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import importlib
13import io
14import os.path
15import stat
16import tempfile
17import unittest
18from typing import cast
20import lsst.resources
21import requests
22import responses
23from lsst.resources import ResourcePath
24from lsst.resources._resourceHandles._httpResourceHandle import HttpReadResourceHandle
25from lsst.resources.http import BearerTokenAuth, SessionStore, _is_protected, _is_webdav_endpoint
26from lsst.resources.tests import GenericTestCase
27from lsst.resources.utils import makeTestTempDir, removeTestTempDir
29TESTDIR = os.path.abspath(os.path.dirname(__file__))
32class GenericHttpTestCase(GenericTestCase, unittest.TestCase):
33 scheme = "http"
34 netloc = "server.example"
37class HttpReadWriteTestCase(unittest.TestCase):
38 """Specialist test cases for WebDAV server.
40 The responses class requires that every possible request be explicitly
41 mocked out. This currently makes it extremely inconvenient to subclass
42 the generic read/write tests shared by other URI schemes. For now use
43 explicit standalone tests.
44 """
46 def setUp(self):
47 # Local test directory
48 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
50 serverRoot = "www.not-exists.orgx"
51 existingFolderName = "existingFolder"
52 existingFileName = "existingFile"
53 notExistingFileName = "notExistingFile"
55 fileForHandleWithRange = "handleWithRange"
56 fileForHandleWithOutRange = "handleWithOutRange"
58 self.baseURL = ResourcePath(f"https://{serverRoot}", forceDirectory=True)
59 self.existingFileResourcePath = ResourcePath(
60 f"https://{serverRoot}/{existingFolderName}/{existingFileName}"
61 )
62 self.notExistingFileResourcePath = ResourcePath(
63 f"https://{serverRoot}/{existingFolderName}/{notExistingFileName}"
64 )
65 self.existingFolderResourcePath = ResourcePath(
66 f"https://{serverRoot}/{existingFolderName}", forceDirectory=True
67 )
68 self.notExistingFolderResourcePath = ResourcePath(
69 f"https://{serverRoot}/{notExistingFileName}", forceDirectory=True
70 )
71 self.handleWithRangeResourcePath = ResourcePath(
72 f"https://{serverRoot}/{existingFolderName}/{fileForHandleWithRange}"
73 )
74 self.handleWithOutRangeResourcePath = ResourcePath(
75 f"https://{serverRoot}/{existingFolderName}/{fileForHandleWithOutRange}"
76 )
78 # Used by the handle tests
79 responses.add(
80 responses.HEAD,
81 self.handleWithRangeResourcePath.geturl(),
82 status=200,
83 headers={"Content-Length": "1024", "Accept-Ranges": "true"},
84 )
85 self.handleWithRangeBody = "These are some \n bytes to read"
86 responses.add(
87 responses.GET,
88 self.handleWithRangeResourcePath.geturl(),
89 status=206,
90 body=self.handleWithRangeBody.encode(),
91 )
92 responses.add(
93 responses.PUT,
94 self.handleWithRangeResourcePath.geturl(),
95 status=201,
96 )
98 responses.add(
99 responses.HEAD,
100 self.handleWithOutRangeResourcePath.geturl(),
101 status=200,
102 headers={"Content-Length": "1024"},
103 )
104 responses.add(
105 responses.GET,
106 self.handleWithOutRangeResourcePath.geturl(),
107 status=200,
108 body="These are some bytes to read".encode(),
109 )
111 # Need to declare the options
112 responses.add(responses.OPTIONS, self.baseURL.geturl(), status=200, headers={"DAV": "1,2,3"})
114 # Used by HttpResourcePath.exists()
115 responses.add(
116 responses.HEAD,
117 self.existingFileResourcePath.geturl(),
118 status=200,
119 headers={"Content-Length": "1024"},
120 )
121 responses.add(responses.HEAD, self.notExistingFileResourcePath.geturl(), status=404)
123 # Used by HttpResourcePath.read()
124 responses.add(
125 responses.GET, self.existingFileResourcePath.geturl(), status=200, body=str.encode("It works!")
126 )
127 responses.add(responses.GET, self.notExistingFileResourcePath.geturl(), status=404)
129 # Used by HttpResourcePath.write()
130 responses.add(responses.PUT, self.existingFileResourcePath.geturl(), status=201)
132 # Used by HttpResourcePath.transfer_from()
133 responses.add(
134 responses.Response(
135 url=self.existingFileResourcePath.geturl(),
136 method="COPY",
137 headers={"Destination": self.existingFileResourcePath.geturl()},
138 status=201,
139 )
140 )
141 responses.add(
142 responses.Response(
143 url=self.existingFileResourcePath.geturl(),
144 method="COPY",
145 headers={"Destination": self.notExistingFileResourcePath.geturl()},
146 status=201,
147 )
148 )
149 responses.add(
150 responses.Response(
151 url=self.existingFileResourcePath.geturl(),
152 method="MOVE",
153 headers={"Destination": self.notExistingFileResourcePath.geturl()},
154 status=201,
155 )
156 )
158 # Used by HttpResourcePath.remove()
159 responses.add(responses.DELETE, self.existingFileResourcePath.geturl(), status=200)
160 responses.add(responses.DELETE, self.notExistingFileResourcePath.geturl(), status=404)
162 # Used by HttpResourcePath.mkdir()
163 responses.add(
164 responses.HEAD,
165 self.existingFolderResourcePath.geturl(),
166 status=200,
167 headers={"Content-Length": "1024"},
168 )
169 responses.add(responses.HEAD, self.baseURL.geturl(), status=200, headers={"Content-Length": "1024"})
170 responses.add(responses.HEAD, self.notExistingFolderResourcePath.geturl(), status=404)
171 responses.add(
172 responses.Response(url=self.notExistingFolderResourcePath.geturl(), method="MKCOL", status=201)
173 )
174 responses.add(
175 responses.Response(url=self.existingFolderResourcePath.geturl(), method="MKCOL", status=403)
176 )
178 # Used by HttpResourcePath._do_put()
179 self.redirectPathNoExpect = ResourcePath(f"https://{serverRoot}/redirect-no-expect/file")
180 self.redirectPathExpect = ResourcePath(f"https://{serverRoot}/redirect-expect/file")
181 redirected_url = f"https://{serverRoot}/redirect/location"
182 responses.add(
183 responses.PUT,
184 self.redirectPathNoExpect.geturl(),
185 headers={"Location": redirected_url},
186 status=307,
187 )
188 responses.add(
189 responses.PUT,
190 self.redirectPathExpect.geturl(),
191 headers={"Location": redirected_url},
192 status=307,
193 match=[responses.matchers.header_matcher({"Content-Length": "0", "Expect": "100-continue"})],
194 )
195 responses.add(responses.PUT, redirected_url, status=202)
197 def tearDown(self):
198 if self.tmpdir:
199 if self.tmpdir.isLocal:
200 removeTestTempDir(self.tmpdir.ospath)
202 @responses.activate
203 def test_file_handle(self):
204 # Test that without the correct header the default method is used.
205 with self.handleWithOutRangeResourcePath.open("rb") as handle:
206 self.assertIsInstance(handle, io.BytesIO)
208 # Test that with correct header the correct handle is returned.
209 with self.handleWithRangeResourcePath.open("rb") as handle:
210 self.assertIsInstance(handle, HttpReadResourceHandle)
212 # Test reading byte ranges works
213 with self.handleWithRangeResourcePath.open("rb") as handle:
214 handle = cast(HttpReadResourceHandle, handle)
215 # This is not a real test, because responses can not actually
216 # handle reading sub byte ranges, so the whole thing needs to be
217 # read.
218 result = handle.read(len(self.handleWithRangeBody)).decode()
219 self.assertEqual(result, self.handleWithRangeBody)
220 # Verify there is no internal buffer.
221 self.assertIsNone(handle._completeBuffer)
222 # Verify the position.
223 self.assertEqual(handle.tell(), len(self.handleWithRangeBody))
225 # Jump back to the beginning and test if reading the whole file
226 # prompts the internal buffer to be read.
227 handle.seek(0)
228 self.assertEqual(handle.tell(), 0)
229 result = handle.read().decode()
230 self.assertIsNotNone(handle._completeBuffer)
231 self.assertEqual(result, self.handleWithRangeBody)
233 # Verify reading as a string handle works as expected.
234 with self.handleWithRangeResourcePath.open("r") as handle:
235 self.assertIsInstance(handle, io.TextIOWrapper)
237 handle = cast(io.TextIOWrapper, handle)
238 self.assertIsInstance(handle.buffer, HttpReadResourceHandle)
240 # Check if string methods work.
241 result = handle.read()
242 self.assertEqual(result, self.handleWithRangeBody)
244 # Verify that write modes invoke the default base method
245 with self.handleWithRangeResourcePath.open("w") as handle:
246 self.assertIsInstance(handle, io.StringIO)
248 @responses.activate
249 def test_exists(self):
251 self.assertTrue(self.existingFileResourcePath.exists())
252 self.assertFalse(self.notExistingFileResourcePath.exists())
254 self.assertEqual(self.existingFileResourcePath.size(), 1024)
255 with self.assertRaises(FileNotFoundError):
256 self.notExistingFileResourcePath.size()
258 @responses.activate
259 def test_remove(self):
261 self.assertIsNone(self.existingFileResourcePath.remove())
262 with self.assertRaises(FileNotFoundError):
263 self.notExistingFileResourcePath.remove()
265 url = "https://example.org/delete"
266 responses.add(responses.DELETE, url, status=404)
267 with self.assertRaises(FileNotFoundError):
268 ResourcePath(url).remove()
270 @responses.activate
271 def test_mkdir(self):
273 # The mock means that we can't check this now exists
274 self.notExistingFolderResourcePath.mkdir()
276 # This should do nothing
277 self.existingFolderResourcePath.mkdir()
279 with self.assertRaises(ValueError):
280 self.notExistingFileResourcePath.mkdir()
282 @responses.activate
283 def test_read(self):
285 self.assertEqual(self.existingFileResourcePath.read().decode(), "It works!")
286 self.assertNotEqual(self.existingFileResourcePath.read().decode(), "Nope.")
287 with self.assertRaises(FileNotFoundError):
288 self.notExistingFileResourcePath.read()
290 # Run this twice to ensure use of cache in code coverage.
291 for _ in (1, 2):
292 with self.existingFileResourcePath.as_local() as local_uri:
293 self.assertTrue(local_uri.isLocal)
294 content = local_uri.read().decode()
295 self.assertEqual(content, "It works!")
297 # Check that the environment variable is being read.
298 lsst.resources.http._TMPDIR = None
299 with unittest.mock.patch.dict(os.environ, {"LSST_RESOURCES_TMPDIR": self.tmpdir.ospath}):
300 with self.existingFileResourcePath.as_local() as local_uri:
301 self.assertTrue(local_uri.isLocal)
302 content = local_uri.read().decode()
303 self.assertEqual(content, "It works!")
304 self.assertIsNotNone(local_uri.relative_to(self.tmpdir))
306 @responses.activate
307 def test_write(self):
309 self.assertIsNone(self.existingFileResourcePath.write(data=str.encode("Some content.")))
310 with self.assertRaises(FileExistsError):
311 self.existingFileResourcePath.write(data=str.encode("Some content."), overwrite=False)
313 url = "https://example.org/put"
314 responses.add(responses.PUT, url, status=404)
315 with self.assertRaises(ValueError):
316 ResourcePath(url).write(data=str.encode("Some content."))
318 @responses.activate
319 def test_do_put_with_redirection(self):
321 # Without LSST_HTTP_PUT_SEND_EXPECT_HEADER.
322 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
323 importlib.reload(lsst.resources.http)
324 body = str.encode("any contents")
325 self.assertIsNone(self.redirectPathNoExpect._do_put(data=body))
327 # With LSST_HTTP_PUT_SEND_EXPECT_HEADER.
328 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "True"}, clear=True):
329 importlib.reload(lsst.resources.http)
330 self.assertIsNone(self.redirectPathExpect._do_put(data=body))
332 @responses.activate
333 def test_transfer(self):
335 # Transferring to self should be no-op.
336 self.existingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
338 self.assertIsNone(self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath))
339 # Should test for existence.
340 # self.assertTrue(self.notExistingFileResourcePath.exists())
342 # Should delete and try again with move.
343 # self.notExistingFileResourcePath.remove()
344 self.assertIsNone(
345 self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath, transfer="move")
346 )
347 # Should then check that it was moved.
348 # self.assertFalse(self.existingFileResourcePath.exists())
350 # Existing file resource should have been removed so this should
351 # trigger FileNotFoundError.
352 # with self.assertRaises(FileNotFoundError):
353 # self.notExistingFileResourcePath.transfer_from(src=self.existingFileResourcePath)
354 with self.assertRaises(ValueError):
355 self.notExistingFileResourcePath.transfer_from(
356 src=self.existingFileResourcePath, transfer="unsupported"
357 )
359 def test_parent(self):
361 self.assertEqual(
362 self.existingFolderResourcePath.geturl(), self.notExistingFileResourcePath.parent().geturl()
363 )
364 self.assertEqual(self.baseURL.geturl(), self.baseURL.parent().geturl())
365 self.assertEqual(
366 self.existingFileResourcePath.parent().geturl(), self.existingFileResourcePath.dirname().geturl()
367 )
369 def test_send_expect_header(self):
371 # Ensure _SEND_EXPECT_HEADER_ON_PUT is correctly initialized from
372 # the environment.
373 os.environ.pop("LSST_HTTP_PUT_SEND_EXPECT_HEADER", None)
374 importlib.reload(lsst.resources.http)
375 self.assertFalse(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
377 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_PUT_SEND_EXPECT_HEADER": "true"}, clear=True):
378 importlib.reload(lsst.resources.http)
379 self.assertTrue(lsst.resources.http._SEND_EXPECT_HEADER_ON_PUT)
381 def test_timeout(self):
383 connect_timeout = 100
384 read_timeout = 200
385 with unittest.mock.patch.dict(
386 os.environ,
387 {"LSST_HTTP_TIMEOUT_CONNECT": str(connect_timeout), "LSST_HTTP_TIMEOUT_READ": str(read_timeout)},
388 clear=True,
389 ):
390 # Force module reload to initialize TIMEOUT.
391 importlib.reload(lsst.resources.http)
392 self.assertEqual(lsst.resources.http.TIMEOUT, (connect_timeout, read_timeout))
394 def test_is_protected(self):
396 self.assertFalse(_is_protected("/this-file-does-not-exist"))
398 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
399 f.write("XXXX")
400 file_path = f.name
402 os.chmod(file_path, stat.S_IRUSR)
403 self.assertTrue(_is_protected(file_path))
405 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
406 os.chmod(file_path, stat.S_IRUSR | mode)
407 self.assertFalse(_is_protected(file_path))
410class WebdavUtilsTestCase(unittest.TestCase):
411 """Test for the Webdav related utilities."""
413 serverRoot = "www.lsstwithwebdav.orgx"
414 wrongRoot = "www.lsstwithoutwebdav.org"
416 def setUp(self):
417 responses.add(responses.OPTIONS, f"https://{self.serverRoot}", status=200, headers={"DAV": "1,2,3"})
418 responses.add(responses.OPTIONS, f"https://{self.wrongRoot}", status=200)
420 @responses.activate
421 def test_is_webdav_endpoint(self):
423 self.assertTrue(_is_webdav_endpoint(f"https://{self.serverRoot}"))
424 self.assertFalse(_is_webdav_endpoint(f"https://{self.wrongRoot}"))
427class BearerTokenAuthTestCase(unittest.TestCase):
428 """Test for the BearerTokenAuth class."""
430 def setUp(self):
431 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
432 self.token = "ABCDE1234"
434 def tearDown(self):
435 if self.tmpdir and self.tmpdir.isLocal:
436 removeTestTempDir(self.tmpdir.ospath)
438 def test_empty_token(self):
439 """Ensure that when no token is provided the request is not
440 modified.
441 """
442 auth = BearerTokenAuth(None)
443 auth._refresh()
444 self.assertIsNone(auth._token)
445 self.assertIsNone(auth._path)
446 req = requests.Request("GET", "https://example.org")
447 self.assertEqual(auth(req), req)
449 def test_token_value(self):
450 """Ensure that when a token value is provided, the 'Authorization'
451 header is added to the requests.
452 """
453 auth = BearerTokenAuth(self.token)
454 req = auth(requests.Request("GET", "https://example.org").prepare())
455 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
457 def test_token_file(self):
458 """Ensure when the provided token is a file path, its contents is
459 correctly used in the the 'Authorization' header of the requests.
460 """
461 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
462 f.write(self.token)
463 token_file_path = f.name
465 # Ensure the request's "Authorization" header is set with the right
466 # token value
467 os.chmod(token_file_path, stat.S_IRUSR)
468 auth = BearerTokenAuth(token_file_path)
469 req = auth(requests.Request("GET", "https://example.org").prepare())
470 self.assertEqual(req.headers.get("Authorization"), f"Bearer {self.token}")
472 # Ensure an exception is raised if either group or other can read the
473 # token file
474 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
475 os.chmod(token_file_path, stat.S_IRUSR | mode)
476 with self.assertRaises(PermissionError):
477 BearerTokenAuth(token_file_path)
480class SessionStoreTestCase(unittest.TestCase):
481 """Test for the SessionStore class."""
483 def setUp(self):
484 self.tmpdir = ResourcePath(makeTestTempDir(TESTDIR))
485 self.rpath = ResourcePath("https://example.org")
487 def tearDown(self):
488 if self.tmpdir and self.tmpdir.isLocal:
489 removeTestTempDir(self.tmpdir.ospath)
491 def test_ca_cert_bundle(self):
492 """Ensure a certificate authorities bundle is used to authentify
493 the remote server.
494 """
495 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
496 f.write("CERT BUNDLE")
497 cert_bundle = f.name
499 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_CACERT_BUNDLE": cert_bundle}, clear=True):
500 session = SessionStore().get(self.rpath)
501 self.assertEqual(session.verify, cert_bundle)
503 def test_user_cert(self):
504 """Ensure if user certificate and private key are provided, they are
505 used for authenticating the client.
506 """
508 # Create mock certificate and private key files.
509 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
510 f.write("CERT")
511 client_cert = f.name
513 with tempfile.NamedTemporaryFile(mode="wt", dir=self.tmpdir.ospath, delete=False) as f:
514 f.write("KEY")
515 client_key = f.name
517 # Check both LSST_HTTP_AUTH_CLIENT_CERT and LSST_HTTP_AUTH_CLIENT_KEY
518 # must be initialized.
519 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert}, clear=True):
520 with self.assertRaises(ValueError):
521 SessionStore().get(self.rpath)
523 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_CLIENT_KEY": client_key}, clear=True):
524 with self.assertRaises(ValueError):
525 SessionStore().get(self.rpath)
527 # Check private key file must be accessible only by its owner.
528 with unittest.mock.patch.dict(
529 os.environ,
530 {"LSST_HTTP_AUTH_CLIENT_CERT": client_cert, "LSST_HTTP_AUTH_CLIENT_KEY": client_key},
531 clear=True,
532 ):
533 # Ensure the session client certificate is initialized when
534 # only the owner can read the private key file.
535 os.chmod(client_key, stat.S_IRUSR)
536 session = SessionStore().get(self.rpath)
537 self.assertEqual(session.cert[0], client_cert)
538 self.assertEqual(session.cert[1], client_key)
540 # Ensure an exception is raised if either group or other can access
541 # the private key file.
542 for mode in (stat.S_IRGRP, stat.S_IWGRP, stat.S_IXGRP, stat.S_IROTH, stat.S_IWOTH, stat.S_IXOTH):
543 os.chmod(client_key, stat.S_IRUSR | mode)
544 with self.assertRaises(PermissionError):
545 SessionStore().get(self.rpath)
547 def test_token_env(self):
548 """Ensure when the token is provided via an environment variable
549 the sessions are equipped with a BearerTokenAuth.
550 """
551 token = "ABCDE"
552 with unittest.mock.patch.dict(os.environ, {"LSST_HTTP_AUTH_BEARER_TOKEN": token}, clear=True):
553 session = SessionStore().get(self.rpath)
554 self.assertEqual(type(session.auth), lsst.resources.http.BearerTokenAuth)
555 self.assertEqual(session.auth._token, token)
556 self.assertIsNone(session.auth._path)
558 def test_sessions(self):
559 """Ensure the session caching mechanism works."""
561 # Ensure the store provides a session for a given URL
562 root_url = "https://example.org"
563 store = SessionStore()
564 session = store.get(ResourcePath(root_url))
565 self.assertIsNotNone(session)
567 # Ensure the sessions retrieved from a single store with the same
568 # root URIs are equal
569 for u in (f"{root_url}", f"{root_url}/path/to/file"):
570 self.assertEqual(session, store.get(ResourcePath(u)))
572 # Ensure sessions retrieved for different root URIs are different
573 another_url = "https://another.example.org"
574 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
576 # Ensure the sessions retrieved from a single store for URLs with
577 # different port numbers are different
578 root_url_with_port = f"{another_url}:12345"
579 session = store.get(ResourcePath(root_url_with_port))
580 self.assertNotEqual(session, store.get(ResourcePath(another_url)))
582 # Ensure the sessions retrieved from a single store with the same
583 # root URIs (including port numbers) are equal
584 for u in (f"{root_url_with_port}", f"{root_url_with_port}/path/to/file"):
585 self.assertEqual(session, store.get(ResourcePath(u)))
588if __name__ == "__main__": 588 ↛ 589line 588 didn't jump to line 589, because the condition on line 588 was never true
589 unittest.main()