Coverage for tests/test_s3.py: 24%

79 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-15 02:25 -0700

1# This file is part of lsst-resources. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# Use of this source code is governed by a 3-clause BSD-style 

10# license that can be found in the LICENSE file. 

11 

12import unittest 

13 

14from lsst.resources import ResourcePath 

15from lsst.resources.s3utils import clean_test_environment 

16from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase 

17 

18try: 

19 import boto3 

20 import botocore 

21 from moto import mock_s3 

22except ImportError: 

23 boto3 = None 

24 

25 def mock_s3(cls): 

26 """A no-op decorator in case moto mock_s3 can not be imported.""" 

27 return cls 

28 

29 

30class GenericS3TestCase(GenericTestCase, unittest.TestCase): 

31 scheme = "s3" 

32 netloc = "my_bucket" 

33 

34 

35@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

36class S3ReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase): 

37 scheme = "s3" 

38 netloc = "my_2nd_bucket" 

39 

40 mock_s3 = mock_s3() 

41 """The mocked s3 interface from moto.""" 

42 

43 def setUp(self): 

44 # Enable S3 mocking of tests. 

45 self.mock_s3.start() 

46 

47 clean_test_environment(self) 

48 

49 # set up some fake credentials if they do not exist 

50 # self.usingDummyCredentials = setAwsEnvCredentials() 

51 

52 # MOTO needs to know that we expect Bucket bucketname to exist 

53 s3 = boto3.resource("s3") 

54 s3.create_bucket(Bucket=self.netloc) 

55 

56 super().setUp() 

57 

58 def tearDown(self): 

59 s3 = boto3.resource("s3") 

60 bucket = s3.Bucket(self.netloc) 

61 try: 

62 bucket.objects.all().delete() 

63 except botocore.exceptions.ClientError as e: 

64 if e.response["Error"]["Code"] == "404": 

65 # the key was not reachable - pass 

66 pass 

67 else: 

68 raise 

69 

70 bucket = s3.Bucket(self.netloc) 

71 bucket.delete() 

72 

73 # Stop the S3 mock. 

74 self.mock_s3.stop() 

75 

76 super().tearDown() 

77 

78 def test_bucket_fail(self): 

79 # Deliberately create URI with unknown bucket. 

80 uri = ResourcePath("s3://badbucket/something/") 

81 

82 with self.assertRaises(ValueError): 

83 uri.mkdir() 

84 

85 with self.assertRaises(FileNotFoundError): 

86 uri.remove() 

87 

88 def test_transfer_progress(self): 

89 """Test progress bar reporting for upload and download.""" 

90 remote = self.root_uri.join("test.dat") 

91 remote.write(b"42") 

92 with ResourcePath.temporary_uri(suffix=".dat") as tmp: 

93 # Download from S3. 

94 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

95 tmp.transfer_from(remote, transfer="auto") 

96 self.assertRegex("".join(cm.output), r"test\.dat.*100\%") 

97 

98 # Upload to S3. 

99 with self.assertLogs("lsst.resources", level="DEBUG") as cm: 

100 remote.transfer_from(tmp, transfer="auto", overwrite=True) 

101 self.assertRegex("".join(cm.output), rf"{tmp.basename()}.*100\%") 

102 

103 def test_handle(self): 

104 remote = self.root_uri.join("test_handle.dat") 

105 with remote.open("wb") as handle: 

106 self.assertTrue(handle.writable()) 

107 # write 6 megabytes to make sure partial write work 

108 handle.write(6 * 1024 * 1024 * b"a") 

109 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

110 handle.flush() 

111 self.assertGreaterEqual(len(handle._multiPartUpload), 1) 

112 

113 # verify file can't be seeked back 

114 with self.assertRaises(OSError): 

115 handle.seek(0) 

116 

117 # write more bytes 

118 handle.write(1024 * b"c") 

119 

120 # seek back and overwrite 

121 handle.seek(6 * 1024 * 1024) 

122 handle.write(1024 * b"b") 

123 

124 with remote.open("rb") as handle: 

125 self.assertTrue(handle.readable()) 

126 # read the first 6 megabytes 

127 result = handle.read(6 * 1024 * 1024) 

128 self.assertEqual(result, 6 * 1024 * 1024 * b"a") 

129 self.assertEqual(handle.tell(), 6 * 1024 * 1024) 

130 # verify additional read gets the next part 

131 result = handle.read(1024) 

132 self.assertEqual(result, 1024 * b"b") 

133 # see back to the beginning to verify seeking 

134 handle.seek(0) 

135 result = handle.read(1024) 

136 self.assertEqual(result, 1024 * b"a") 

137 

138 

139if __name__ == "__main__": 

140 unittest.main()