Coverage for tests/test_s3utils.py: 30%

62 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-12 10:52 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import os 

23import unittest 

24from unittest import mock 

25 

26from lsst.resources.s3utils import clean_test_environment 

27 

28try: 

29 import boto3 

30 from botocore.exceptions import ParamValidationError 

31 from moto import mock_s3 

32except ImportError: 

33 boto3 = None 

34 

35 def mock_s3(cls): 

36 """No-op decorator in case moto mock_s3 can not be imported.""" 

37 return cls 

38 

39 

40from lsst.resources import ResourcePath 

41from lsst.resources.location import Location 

42from lsst.resources.s3utils import ( 

43 bucketExists, 

44 getS3Client, 

45 s3CheckFileExists, 

46 setAwsEnvCredentials, 

47 unsetAwsEnvCredentials, 

48) 

49 

50 

51@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

52@mock_s3 

53class S3UtilsTestCase(unittest.TestCase): 

54 """Test for the S3 related utilities.""" 

55 

56 bucketName = "test_bucket_name" 

57 fileName = "testFileName" 

58 

59 def setUp(self): 

60 # set up some fake credentials if they do not exist 

61 self.usingDummyCredentials = setAwsEnvCredentials() 

62 

63 clean_test_environment(self) 

64 

65 self.client = getS3Client() 

66 try: 

67 self.client.create_bucket(Bucket=self.bucketName) 

68 self.client.put_object(Bucket=self.bucketName, Key=self.fileName, Body=b"test content") 

69 except self.client.exceptions.BucketAlreadyExists: 

70 pass 

71 

72 def tearDown(self): 

73 objects = self.client.list_objects(Bucket=self.bucketName) 

74 if "Contents" in objects: 

75 for item in objects["Contents"]: 

76 self.client.delete_object(Bucket=self.bucketName, Key=item["Key"]) 

77 

78 self.client.delete_bucket(Bucket=self.bucketName) 

79 

80 # unset any potentially set dummy credentials 

81 if self.usingDummyCredentials: 

82 unsetAwsEnvCredentials() 

83 

84 def testBucketExists(self): 

85 self.assertTrue(bucketExists(f"{self.bucketName}")) 

86 self.assertFalse(bucketExists(f"{self.bucketName}_no_exist")) 

87 

88 def testCephBucket(self): 

89 with mock.patch.dict(os.environ, {"LSST_DISABLE_BUCKET_VALIDATION": "N"}): 

90 self.assertEqual(os.environ["LSST_DISABLE_BUCKET_VALIDATION"], "N") 

91 local_client = getS3Client() 

92 with self.assertRaises(ParamValidationError): 

93 bucketExists("foo:bar", local_client) 

94 with mock.patch.dict(os.environ, {"LSST_DISABLE_BUCKET_VALIDATION": "1"}): 

95 self.assertEqual(os.environ["LSST_DISABLE_BUCKET_VALIDATION"], "1") 

96 local_client = getS3Client() 

97 self.assertFalse(bucketExists("foo:bar", local_client)) 

98 

99 def testFileExists(self): 

100 self.assertTrue(s3CheckFileExists(client=self.client, bucket=self.bucketName, path=self.fileName)[0]) 

101 self.assertFalse( 

102 s3CheckFileExists(client=self.client, bucket=self.bucketName, path=self.fileName + "_NO_EXIST")[0] 

103 ) 

104 

105 datastoreRootUri = f"s3://{self.bucketName}/" 

106 uri = f"s3://{self.bucketName}/{self.fileName}" 

107 

108 buri = ResourcePath(uri) 

109 location = Location(datastoreRootUri, self.fileName) 

110 

111 self.assertTrue(s3CheckFileExists(client=self.client, path=buri)[0]) 

112 # just to make sure the overloaded keyword works correctly 

113 self.assertTrue(s3CheckFileExists(buri, client=self.client)[0]) 

114 self.assertTrue(s3CheckFileExists(client=self.client, path=location)[0]) 

115 

116 # make sure supplying strings resolves correctly too 

117 self.assertTrue(s3CheckFileExists(uri, client=self.client)) 

118 self.assertTrue(s3CheckFileExists(uri)) 

119 

120 

121if __name__ == "__main__": 

122 unittest.main()