Coverage for tests/test_s3.py: 24%

79 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-13 09:44 +0000

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import unittest 

13 

14from lsst.resources import ResourcePath 

15from lsst.resources.s3utils import clean_test_environment 

16from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

17 

18try: 

19 import boto3 

20 import botocore 

21 from moto import mock_s3 

22except ImportError: 

23 boto3 = None 

24 

25 def mock_s3(cls): 

26 """No-op decorator in case moto mock_s3 can not be imported.""" 

27 return cls 

28 

29 

30class GenericS3TestCase(GenericTestCase, unittest.TestCase): 

31 """Generic tests of S3 URIs.""" 

32 

33 scheme = "s3" 

34 netloc = "my_bucket" 

35 

36 

37@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

38class S3ReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase): 

39 """Tests of reading and writing S3 URIs.""" 

40 

41 scheme = "s3" 

42 netloc = "my_2nd_bucket" 

43 

44 mock_s3 = mock_s3() 

45 """The mocked s3 interface from moto.""" 

46 

47 def setUp(self): 

48 # Enable S3 mocking of tests. 

49 self.mock_s3.start() 

50 

51 clean_test_environment(self) 

52 

53 # set up some fake credentials if they do not exist 

54 # self.usingDummyCredentials = setAwsEnvCredentials() 

55 

56 # MOTO needs to know that we expect Bucket bucketname to exist 

57 s3 = boto3.resource("s3") 

58 s3.create_bucket(Bucket=self.netloc) 

59 

60 super().setUp() 

61 

62 def tearDown(self): 

63 s3 = boto3.resource("s3") 

64 bucket = s3.Bucket(self.netloc) 

65 try: 

66 bucket.objects.all().delete() 

67 except botocore.exceptions.ClientError as e: 

68 if e.response["Error"]["Code"] == "404": 

69 # the key was not reachable - pass 

70 pass 

71 else: 

72 raise 

73 

74 bucket = s3.Bucket(self.netloc) 

75 bucket.delete() 

76 

77 # Stop the S3 mock. 

78 self.mock_s3.stop() 

79 

80 super().tearDown() 

81 

82 def test_bucket_fail(self): 

83 # Deliberately create URI with unknown bucket. 

84 uri = ResourcePath("s3://badbucket/something/") 

85 

86 with self.assertRaises(ValueError): 

87 uri.mkdir() 

88 

89 with self.assertRaises(FileNotFoundError): 

90 uri.remove() 

91 

92 def test_transfer_progress(self): 

93 """Test progress bar reporting for upload and download.""" 

94 remote = self.root_uri.join("test.dat") 

95 remote.write(b"42") 

96 with ResourcePath.temporary_uri(suffix=".dat") as tmp: 

97 # Download from S3. 

98 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

99 tmp.transfer_from(remote, transfer="auto") 

100 self.assertRegex("".join(cm.output), r"test\.dat.*100\%") 

101 

102 # Upload to S3. 

103 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

104 remote.transfer_from(tmp, transfer="auto", overwrite=True) 

105 self.assertRegex("".join(cm.output), rf"{tmp.basename()}.*100\%") 

106 

107 def test_handle(self): 

108 remote = self.root_uri.join("test_handle.dat") 

109 with remote.open("wb") as handle: 

110 self.assertTrue(handle.writable()) 

111 # write 6 megabytes to make sure partial write work 

112 handle.write(6 * 1024 * 1024 * b"a") 

113 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

114 handle.flush() 

115 self.assertGreaterEqual(len(handle._multiPartUpload), 1) 

116 

117 # verify file can't be seeked back 

118 with self.assertRaises(OSError): 

119 handle.seek(0) 

120 

121 # write more bytes 

122 handle.write(1024 * b"c") 

123 

124 # seek back and overwrite 

125 handle.seek(6 * 1024 * 1024) 

126 handle.write(1024 * b"b") 

127 

128 with remote.open("rb") as handle: 

129 self.assertTrue(handle.readable()) 

130 # read the first 6 megabytes 

131 result = handle.read(6 * 1024 * 1024) 

132 self.assertEqual(result, 6 * 1024 * 1024 * b"a") 

133 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

134 # verify additional read gets the next part 

135 result = handle.read(1024) 

136 self.assertEqual(result, 1024 * b"b") 

137 # see back to the beginning to verify seeking 

138 handle.seek(0) 

139 result = handle.read(1024) 

140 self.assertEqual(result, 1024 * b"a") 

141 

142 

143if __name__ == "__main__": 

144 unittest.main()