Coverage for python/lsst/daf/butler/core/s3utils.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22__all__ = ("s3CheckFileExists", "bucketExists", "setAwsEnvCredentials",
23 "unsetAwsEnvCredentials")
25import os
27try:
28 import boto3
29except ImportError:
30 boto3 = None
32from .location import ButlerURI, Location
35def s3CheckFileExists(path, bucket=None, client=None):
36 """Returns (True, filesize) if file exists in the bucket and (False, -1) if
37 the file is not found.
39 Parameters
40 ----------
41 path : `Location`, `ButlerURI`, `str`
42 Location or ButlerURI containing the bucket name and filepath.
43 bucket : `str`, optional
44 Name of the bucket in which to look. If provided, path will be assumed
45 to correspond to be relative to the given bucket.
46 client : `boto3.client`, optional
47 S3 Client object to query, if not supplied boto3 will try to resolve
48 the credentials as in order described in its manual_.
50 Returns
51 -------
52 exists : `bool`
53 True if key exists, False otherwise.
54 size : `int`
55 Size of the key, if key exists, in bytes, otherwise -1
57 Notes
58 -----
59 S3 Paths are sensitive to leading and trailing path separators.
61 .. _manual: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/\
62 configuration.html#configuring-credentials
63 """
64 if boto3 is None:
65 raise ModuleNotFoundError(("Could not find boto3. "
66 "Are you sure it is installed?"))
68 if client is None:
69 client = boto3.client('s3')
71 if isinstance(path, str):
72 if bucket is not None:
73 filepath = path
74 else:
75 uri = ButlerURI(path)
76 bucket = uri.netloc
77 filepath = uri.relativeToPathRoot
78 elif isinstance(path, (ButlerURI, Location)):
79 bucket = path.netloc
80 filepath = path.relativeToPathRoot
82 try:
83 obj = client.head_object(Bucket=bucket, Key=filepath)
84 return (True, obj["ContentLength"])
85 except client.exceptions.ClientError as err:
86 # resource unreachable error means key does not exist
87 if err.response["ResponseMetadata"]["HTTPStatusCode"] == 404:
88 return (False, -1)
89 # head_object returns 404 when object does not exist only when user has
90 # s3:ListBucket permission. If list permission does not exist a 403 is
91 # returned. In practical terms this generally means that the file does
92 # not exist, but it could also mean user lacks s3:GetObject permission:
93 # https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
94 # I don't think its possible to discern which case is it with certainty
95 if err.response["ResponseMetadata"]["HTTPStatusCode"] == 403:
96 raise PermissionError("Forbidden HEAD operation error occured. "
97 "Verify s3:ListBucket and s3:GetObject "
98 "permissions are granted for your IAM user. ") from err
99 raise
102def bucketExists(bucketName, client=None):
103 """Check if the S3 bucket with the given name actually exists.
105 Parameters
106 ----------
107 bucketName : `str`
108 Name of the S3 Bucket
109 client : `boto3.client`, optional
110 S3 Client object to query, if not supplied boto3 will try to resolve
111 the credentials as in order described in its manual_.
113 Returns
114 -------
115 exists : `bool`
116 True if it exists, False if no Bucket with specified parameters is
117 found.
119 .. _manual: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/\
120 configuration.html#configuring-credentials
121 """
122 if boto3 is None:
123 raise ModuleNotFoundError(("Could not find boto3. "
124 "Are you sure it is installed?"))
126 s3 = boto3.client("s3")
127 try:
128 s3.get_bucket_location(Bucket=bucketName)
129 return True
130 except s3.exceptions.NoSuchBucket:
131 return False
134def setAwsEnvCredentials(accessKeyId='dummyAccessKeyId', secretAccessKey="dummySecretAccessKey"):
135 """Set AWS credentials environmental variables AWS_ACCESS_KEY_ID and
136 AWS_SECRET_ACCESS_KEY.
138 Parameters
139 ----------
140 accessKeyId : `str`
141 Value given to AWS_ACCESS_KEY_ID environmental variable. Defaults to
142 'dummyAccessKeyId'
143 secretAccessKey : `str`
144 Value given to AWS_SECRET_ACCESS_KEY environmental variable. Defaults
145 to 'dummySecretAccessKey'
147 Returns
148 -------
149 setEnvCredentials : `bool`
150 True when environmental variables were set, False otherwise.
152 Notes
153 -----
154 If either AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are not set, both
155 values are overwritten.
156 """
157 if "AWS_ACCESS_KEY_ID" not in os.environ or "AWS_SECRET_ACCESS_KEY" not in os.environ:
158 os.environ["AWS_ACCESS_KEY_ID"] = accessKeyId
159 os.environ["AWS_SECRET_ACCESS_KEY"] = secretAccessKey
160 return True
161 return False
164def unsetAwsEnvCredentials():
165 """Unsets AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environmental
166 variables.
167 """
168 if "AWS_ACCESS_KEY_ID" in os.environ:
169 del os.environ["AWS_ACCESS_KEY_ID"]
170 if "AWS_SECRET_ACCESS_KEY" in os.environ:
171 del os.environ["AWS_SECRET_ACCESS_KEY"]