Coverage for tests/test_s3.py: 24%

98 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-09 11:30 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import time 

13import unittest 

14from urllib.parse import parse_qs, urlparse 

15 

16from lsst.resources import ResourcePath 

17from lsst.resources.s3utils import clean_test_environment_for_s3 

18from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

19 

20try: 

21 import boto3 

22 import botocore 

23 from moto import mock_s3 

24except ImportError: 

25 boto3 = None 

26 

27 def mock_s3(cls): 

28 """No-op decorator in case moto mock_s3 can not be imported.""" 

29 return cls 

30 

31 

32class GenericS3TestCase(GenericTestCase, unittest.TestCase): 

33 """Generic tests of S3 URIs.""" 

34 

35 scheme = "s3" 

36 netloc = "my_bucket" 

37 

38 

39@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

40class S3ReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase): 

41 """Tests of reading and writing S3 URIs.""" 

42 

43 scheme = "s3" 

44 netloc = "my_2nd_bucket" 

45 

46 mock_s3 = mock_s3() 

47 """The mocked s3 interface from moto.""" 

48 

49 def setUp(self): 

50 self.enterContext(clean_test_environment_for_s3()) 

51 # Enable S3 mocking of tests. 

52 self.mock_s3.start() 

53 

54 # MOTO needs to know that we expect Bucket bucketname to exist 

55 s3 = boto3.resource("s3") 

56 s3.create_bucket(Bucket=self.netloc) 

57 

58 super().setUp() 

59 

60 def tearDown(self): 

61 s3 = boto3.resource("s3") 

62 bucket = s3.Bucket(self.netloc) 

63 try: 

64 bucket.objects.all().delete() 

65 except botocore.exceptions.ClientError as e: 

66 if e.response["Error"]["Code"] == "404": 

67 # the key was not reachable - pass 

68 pass 

69 else: 

70 raise 

71 

72 bucket = s3.Bucket(self.netloc) 

73 bucket.delete() 

74 

75 # Stop the S3 mock. 

76 self.mock_s3.stop() 

77 

78 super().tearDown() 

79 

80 def test_bucket_fail(self): 

81 # Deliberately create URI with unknown bucket. 

82 uri = ResourcePath("s3://badbucket/something/") 

83 

84 with self.assertRaises(ValueError): 

85 uri.mkdir() 

86 

87 with self.assertRaises(FileNotFoundError): 

88 uri.remove() 

89 

90 def test_transfer_progress(self): 

91 """Test progress bar reporting for upload and download.""" 

92 remote = self.root_uri.join("test.dat") 

93 remote.write(b"42") 

94 with ResourcePath.temporary_uri(suffix=".dat") as tmp: 

95 # Download from S3. 

96 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

97 tmp.transfer_from(remote, transfer="auto") 

98 self.assertRegex("".join(cm.output), r"test\.dat.*100\%") 

99 

100 # Upload to S3. 

101 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

102 remote.transfer_from(tmp, transfer="auto", overwrite=True) 

103 self.assertRegex("".join(cm.output), rf"{tmp.basename()}.*100\%") 

104 

105 def test_handle(self): 

106 remote = self.root_uri.join("test_handle.dat") 

107 with remote.open("wb") as handle: 

108 self.assertTrue(handle.writable()) 

109 # write 6 megabytes to make sure partial write work 

110 handle.write(6 * 1024 * 1024 * b"a") 

111 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

112 handle.flush() 

113 self.assertGreaterEqual(len(handle._multiPartUpload), 1) 

114 

115 # verify file can't be seeked back 

116 with self.assertRaises(OSError): 

117 handle.seek(0) 

118 

119 # write more bytes 

120 handle.write(1024 * b"c") 

121 

122 # seek back and overwrite 

123 handle.seek(6 * 1024 * 1024) 

124 handle.write(1024 * b"b") 

125 

126 with remote.open("rb") as handle: 

127 self.assertTrue(handle.readable()) 

128 # read the first 6 megabytes 

129 result = handle.read(6 * 1024 * 1024) 

130 self.assertEqual(result, 6 * 1024 * 1024 * b"a") 

131 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

132 # verify additional read gets the next part 

133 result = handle.read(1024) 

134 self.assertEqual(result, 1024 * b"b") 

135 # see back to the beginning to verify seeking 

136 handle.seek(0) 

137 result = handle.read(1024) 

138 self.assertEqual(result, 1024 * b"a") 

139 

140 def test_url_signing(self): 

141 s3_path = self.root_uri.join("url-signing-test.txt") 

142 

143 put_url = s3_path.generate_presigned_put_url(expiration_time_seconds=1800) 

144 self._check_presigned_url(put_url, 1800) 

145 get_url = s3_path.generate_presigned_get_url(expiration_time_seconds=3600) 

146 self._check_presigned_url(get_url, 3600) 

147 

148 # Moto monkeypatches the 'requests' library to mock access to presigned 

149 # URLs, so we are able to use HttpResourcePath to access the URLs in 

150 # this test 

151 test_data = b"test123" 

152 ResourcePath(put_url).write(test_data) 

153 retrieved = ResourcePath(get_url).read() 

154 self.assertEqual(retrieved, test_data) 

155 

156 def _check_presigned_url(self, url: str, expiration_time_seconds: int): 

157 parsed = urlparse(url) 

158 self.assertEqual(parsed.scheme, "https") 

159 

160 actual_expiration_timestamp = int(parse_qs(parsed.query)["Expires"][0]) 

161 current_time = int(time.time()) 

162 expected_expiration_timestamp = current_time + expiration_time_seconds 

163 # Allow some flex in the expiration time in case this test process goes 

164 # out to lunch for a while on a busy CI machine 

165 self.assertLessEqual(abs(expected_expiration_timestamp - actual_expiration_timestamp), 120) 

166 

167 

168if __name__ == "__main__": 

169 unittest.main()