Coverage for tests/test_s3.py: 24%
98 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-09 11:30 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-09 11:30 +0000
1# This file is part of lsst-resources.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# Use of this source code is governed by a 3-clause BSD-style
10# license that can be found in the LICENSE file.
12import time
13import unittest
14from urllib.parse import parse_qs, urlparse
16from lsst.resources import ResourcePath
17from lsst.resources.s3utils import clean_test_environment_for_s3
18from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase
20try:
21 import boto3
22 import botocore
23 from moto import mock_s3
24except ImportError:
25 boto3 = None
27 def mock_s3(cls):
28 """No-op decorator in case moto mock_s3 can not be imported."""
29 return cls
32class GenericS3TestCase(GenericTestCase, unittest.TestCase):
33 """Generic tests of S3 URIs."""
35 scheme = "s3"
36 netloc = "my_bucket"
39@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
40class S3ReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase):
41 """Tests of reading and writing S3 URIs."""
43 scheme = "s3"
44 netloc = "my_2nd_bucket"
46 mock_s3 = mock_s3()
47 """The mocked s3 interface from moto."""
49 def setUp(self):
50 self.enterContext(clean_test_environment_for_s3())
51 # Enable S3 mocking of tests.
52 self.mock_s3.start()
54 # MOTO needs to know that we expect Bucket bucketname to exist
55 s3 = boto3.resource("s3")
56 s3.create_bucket(Bucket=self.netloc)
58 super().setUp()
60 def tearDown(self):
61 s3 = boto3.resource("s3")
62 bucket = s3.Bucket(self.netloc)
63 try:
64 bucket.objects.all().delete()
65 except botocore.exceptions.ClientError as e:
66 if e.response["Error"]["Code"] == "404":
67 # the key was not reachable - pass
68 pass
69 else:
70 raise
72 bucket = s3.Bucket(self.netloc)
73 bucket.delete()
75 # Stop the S3 mock.
76 self.mock_s3.stop()
78 super().tearDown()
80 def test_bucket_fail(self):
81 # Deliberately create URI with unknown bucket.
82 uri = ResourcePath("s3://badbucket/something/")
84 with self.assertRaises(ValueError):
85 uri.mkdir()
87 with self.assertRaises(FileNotFoundError):
88 uri.remove()
90 def test_transfer_progress(self):
91 """Test progress bar reporting for upload and download."""
92 remote = self.root_uri.join("test.dat")
93 remote.write(b"42")
94 with ResourcePath.temporary_uri(suffix=".dat") as tmp:
95 # Download from S3.
96 with self.assertLogs("lsst.resources", level="DEBUG") as cm:
97 tmp.transfer_from(remote, transfer="auto")
98 self.assertRegex("".join(cm.output), r"test\.dat.*100\%")
100 # Upload to S3.
101 with self.assertLogs("lsst.resources", level="DEBUG") as cm:
102 remote.transfer_from(tmp, transfer="auto", overwrite=True)
103 self.assertRegex("".join(cm.output), rf"{tmp.basename()}.*100\%")
105 def test_handle(self):
106 remote = self.root_uri.join("test_handle.dat")
107 with remote.open("wb") as handle:
108 self.assertTrue(handle.writable())
109 # write 6 megabytes to make sure partial write work
110 handle.write(6 * 1024 * 1024 * b"a")
111 self.assertEqual(handle.tell(), 6 * 1024 * 1024)
112 handle.flush()
113 self.assertGreaterEqual(len(handle._multiPartUpload), 1)
115 # verify file can't be seeked back
116 with self.assertRaises(OSError):
117 handle.seek(0)
119 # write more bytes
120 handle.write(1024 * b"c")
122 # seek back and overwrite
123 handle.seek(6 * 1024 * 1024)
124 handle.write(1024 * b"b")
126 with remote.open("rb") as handle:
127 self.assertTrue(handle.readable())
128 # read the first 6 megabytes
129 result = handle.read(6 * 1024 * 1024)
130 self.assertEqual(result, 6 * 1024 * 1024 * b"a")
131 self.assertEqual(handle.tell(), 6 * 1024 * 1024)
132 # verify additional read gets the next part
133 result = handle.read(1024)
134 self.assertEqual(result, 1024 * b"b")
135 # see back to the beginning to verify seeking
136 handle.seek(0)
137 result = handle.read(1024)
138 self.assertEqual(result, 1024 * b"a")
140 def test_url_signing(self):
141 s3_path = self.root_uri.join("url-signing-test.txt")
143 put_url = s3_path.generate_presigned_put_url(expiration_time_seconds=1800)
144 self._check_presigned_url(put_url, 1800)
145 get_url = s3_path.generate_presigned_get_url(expiration_time_seconds=3600)
146 self._check_presigned_url(get_url, 3600)
148 # Moto monkeypatches the 'requests' library to mock access to presigned
149 # URLs, so we are able to use HttpResourcePath to access the URLs in
150 # this test
151 test_data = b"test123"
152 ResourcePath(put_url).write(test_data)
153 retrieved = ResourcePath(get_url).read()
154 self.assertEqual(retrieved, test_data)
156 def _check_presigned_url(self, url: str, expiration_time_seconds: int):
157 parsed = urlparse(url)
158 self.assertEqual(parsed.scheme, "https")
160 actual_expiration_timestamp = int(parse_qs(parsed.query)["Expires"][0])
161 current_time = int(time.time())
162 expected_expiration_timestamp = current_time + expiration_time_seconds
163 # Allow some flex in the expiration time in case this test process goes
164 # out to lunch for a while on a busy CI machine
165 self.assertLessEqual(abs(expected_expiration_timestamp - actual_expiration_timestamp), 120)
168if __name__ == "__main__":
169 unittest.main()