Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import unittest 

23 

24try: 

25 import boto3 

26 import botocore 

27 from moto import mock_s3 

28except ImportError: 

29 boto3 = None 

30 

31 def mock_s3(cls): 

32 """A no-op decorator in case moto mock_s3 can not be imported. 

33 """ 

34 return cls 

35 

36from lsst.daf.butler.core.s3utils import (bucketExists, s3CheckFileExists, 

37 setAwsEnvCredentials, unsetAwsEnvCredentials) 

38from lsst.daf.butler.core.location import Location, ButlerURI 

39 

40 

41@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!") 

42@mock_s3 

43class S3UtilsTestCase(unittest.TestCase): 

44 """Test for the S3 related utilities. 

45 """ 

46 bucketName = "testBucketName" 

47 fileName = "testFileName" 

48 

49 def setUp(self): 

50 # set up some fake credentials if they do not exist 

51 self.usingDummyCredentials = setAwsEnvCredentials() 

52 

53 s3 = boto3.client("s3") 

54 try: 

55 s3.create_bucket(Bucket=self.bucketName) 

56 s3.put_object(Bucket=self.bucketName, Key=self.fileName, 

57 Body=b"test content") 

58 except s3.exceptions.BucketAlreadyExists: 

59 pass 

60 

61 def tearDown(self): 

62 s3 = boto3.resource('s3') 

63 bucket = s3.Bucket(self.bucketName) 

64 try: 

65 bucket.objects.all().delete() 

66 except botocore.exceptions.ClientError as err: 

67 errorcode = err.response["ResponseMetadata"]["HTTPStatusCode"] 

68 if errorcode == 404: 

69 # the key does not exists - pass 

70 pass 

71 else: 

72 raise 

73 

74 bucket = s3.Bucket(self.bucketName) 

75 bucket.delete() 

76 

77 # unset any potentially set dummy credentials 

78 if self.usingDummyCredentials: 

79 unsetAwsEnvCredentials() 

80 

81 def testBucketExists(self): 

82 self.assertTrue(bucketExists(f"{self.bucketName}")) 

83 self.assertFalse(bucketExists(f"{self.bucketName}_NO_EXIST")) 

84 

85 def testFileExists(self): 

86 s3 = boto3.client('s3') 

87 self.assertTrue(s3CheckFileExists(client=s3, bucket=self.bucketName, 

88 path=self.fileName)[0]) 

89 self.assertFalse(s3CheckFileExists(client=s3, bucket=self.bucketName, 

90 path=self.fileName+"_NO_EXIST")[0]) 

91 

92 datastoreRootUri = f"s3://{self.bucketName}/" 

93 uri = f"s3://{self.bucketName}/{self.fileName}" 

94 

95 buri = ButlerURI(uri) 

96 location = Location(datastoreRootUri, self.fileName) 

97 

98 self.assertTrue(s3CheckFileExists(client=s3, path=buri)[0]) 

99 # just to make sure the overloaded keyword works correctly 

100 self.assertTrue(s3CheckFileExists(buri, client=s3)[0]) 

101 self.assertTrue(s3CheckFileExists(client=s3, path=location)[0]) 

102 

103 # make sure supplying strings resolves correctly too 

104 self.assertTrue(s3CheckFileExists(uri, client=s3)) 

105 self.assertTrue(s3CheckFileExists(uri)) 

106 

107 

108if __name__ == "__main__": 108 ↛ 109line 108 didn't jump to line 109, because the condition on line 108 was never true

109 unittest.main()