Coverage for tests/test_s3utils.py : 31%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import unittest
24try:
25 import boto3
26 import botocore
27 from moto import mock_s3
28except ImportError:
29 boto3 = None
31 def mock_s3(cls):
32 """A no-op decorator in case moto mock_s3 can not be imported.
33 """
34 return cls
36from lsst.daf.butler.core.s3utils import (bucketExists, s3CheckFileExists,
37 setAwsEnvCredentials, unsetAwsEnvCredentials)
38from lsst.daf.butler.core.location import Location, ButlerURI
41@unittest.skipIf(not boto3, "Warning: boto3 AWS SDK not found!")
42@mock_s3
43class S3UtilsTestCase(unittest.TestCase):
44 """Test for the S3 related utilities.
45 """
46 bucketName = "testBucketName"
47 fileName = "testFileName"
49 def setUp(self):
50 # set up some fake credentials if they do not exist
51 self.usingDummyCredentials = setAwsEnvCredentials()
53 s3 = boto3.client("s3")
54 try:
55 s3.create_bucket(Bucket=self.bucketName)
56 s3.put_object(Bucket=self.bucketName, Key=self.fileName,
57 Body=b"test content")
58 except s3.exceptions.BucketAlreadyExists:
59 pass
61 def tearDown(self):
62 s3 = boto3.resource('s3')
63 bucket = s3.Bucket(self.bucketName)
64 try:
65 bucket.objects.all().delete()
66 except botocore.exceptions.ClientError as err:
67 errorcode = err.response["ResponseMetadata"]["HTTPStatusCode"]
68 if errorcode == 404:
69 # the key does not exists - pass
70 pass
71 else:
72 raise
74 bucket = s3.Bucket(self.bucketName)
75 bucket.delete()
77 # unset any potentially set dummy credentials
78 if self.usingDummyCredentials:
79 unsetAwsEnvCredentials()
81 def testBucketExists(self):
82 self.assertTrue(bucketExists(f"{self.bucketName}"))
83 self.assertFalse(bucketExists(f"{self.bucketName}_NO_EXIST"))
85 def testFileExists(self):
86 s3 = boto3.client('s3')
87 self.assertTrue(s3CheckFileExists(client=s3, bucket=self.bucketName,
88 path=self.fileName)[0])
89 self.assertFalse(s3CheckFileExists(client=s3, bucket=self.bucketName,
90 path=self.fileName+"_NO_EXIST")[0])
92 datastoreRootUri = f"s3://{self.bucketName}/"
93 uri = f"s3://{self.bucketName}/{self.fileName}"
95 buri = ButlerURI(uri)
96 location = Location(datastoreRootUri, self.fileName)
98 self.assertTrue(s3CheckFileExists(client=s3, path=buri)[0])
99 # just to make sure the overloaded keyword works correctly
100 self.assertTrue(s3CheckFileExists(buri, client=s3)[0])
101 self.assertTrue(s3CheckFileExists(client=s3, path=location)[0])
103 # make sure supplying strings resolves correctly too
104 self.assertTrue(s3CheckFileExists(uri, client=s3))
105 self.assertTrue(s3CheckFileExists(uri))
108if __name__ == "__main__": 108 ↛ 109line 108 didn't jump to line 109, because the condition on line 108 was never true
109 unittest.main()