Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("getHttpSession", "isWebdavEndpoint", "webdavCheckFileExists", 

25 "folderExists", "webdavDeleteFile") 

26 

27import os 

28import requests 

29import logging 

30from requests.structures import CaseInsensitiveDict 

31 

32from typing import ( 

33 Optional, 

34 Tuple, 

35 Union, 

36) 

37 

38from .location import ButlerURI, Location 

39 

40log = logging.getLogger(__name__) 

41 

42 

43def getHttpSession() -> requests.Session: 

44 """Create a requests.Session pre-configured with environment variable data 

45 

46 Returns 

47 ------- 

48 session : `requests.Session` 

49 An http session used to execute requests. 

50 

51 Notes 

52 ----- 

53 The WEBDAV_AUTH_METHOD must be set to obtain a session. 

54 Depending on the chosen method, additional 

55 environment variables are required: 

56 

57 X509: must set WEBDAV_PROXY_CERT 

58 (path to proxy certificate used to authenticate requests) 

59 

60 TOKEN: must set WEBDAV_BEARER_TOKEN 

61 (bearer token used to authenticate requests, as a single string) 

62 

63 NB: requests will read CA certificates in REQUESTS_CA_BUNDLE 

64 It must be manually exported according to the system CA directory. 

65 """ 

66 session = requests.Session() 

67 log.debug("Creating new HTTP session...") 

68 

69 try: 

70 env_auth_method = os.environ['WEBDAV_AUTH_METHOD'] 

71 except KeyError: 

72 raise KeyError("Environment variable WEBDAV_AUTH_METHOD is not set, please use values X509 or TOKEN") 

73 

74 if env_auth_method == "X509": 

75 try: 

76 proxy_cert = os.environ['WEBDAV_PROXY_CERT'] 

77 except KeyError: 

78 raise KeyError("Environment variable WEBDAV_PROXY_CERT is not set") 

79 log.debug("... using x509 authentication.") 

80 session.cert = (proxy_cert, proxy_cert) 

81 elif env_auth_method == "TOKEN": 

82 try: 

83 bearer_token = os.environ['WEBDAV_BEARER_TOKEN'] 

84 except KeyError: 

85 raise KeyError("Environment variable WEBDAV_BEARER_TOKEN is not set") 

86 log.debug("... using bearer-token authentication.") 

87 session.headers = CaseInsensitiveDict({'Authorization': 'Bearer ' + bearer_token}) 

88 else: 

89 raise ValueError("Environment variable WEBDAV_AUTH_METHOD must be set to X509 or TOKEN") 

90 

91 log.debug("Session configured and ready.") 

92 

93 return session 

94 

95 

96def webdavCheckFileExists(path: Union[Location, ButlerURI, str], 

97 session: Optional[requests.Session] = None) -> Tuple[bool, int]: 

98 """Check that a remote HTTP resource exists. 

99 

100 Parameters 

101 ---------- 

102 path : `Location`, `ButlerURI` or `str` 

103 Location or ButlerURI containing the bucket name and filepath. 

104 session : `requests.Session`, optional 

105 Session object to query. 

106 

107 Returns 

108 ------- 

109 exists : `bool` 

110 True if resource exists, False otherwise. 

111 size : `int` 

112 Size of the resource, if it exists, in bytes, otherwise -1 

113 """ 

114 if session is None: 

115 session = getHttpSession() 

116 

117 filepath = _getFileURL(path) 

118 

119 log.debug("Checking if file exists: %s", filepath) 

120 

121 r = session.head(filepath) 

122 return (True, int(r.headers['Content-Length'])) if r.status_code == 200 else (False, -1) 

123 

124 

125def webdavDeleteFile(path: Union[Location, ButlerURI, str], 

126 session: Optional[requests.Session] = None) -> None: 

127 """Remove a remote HTTP resource. 

128 Raises a FileNotFoundError if the resource does not exist or on failure. 

129 

130 Parameters 

131 ---------- 

132 path : `Location`, `ButlerURI` or `str` 

133 Location or ButlerURI containing the bucket name and filepath. 

134 session : `requests.Session`, optional 

135 Session object to query. 

136 """ 

137 if session is None: 

138 session = getHttpSession() 

139 

140 filepath = _getFileURL(path) 

141 

142 log.debug("Removing file: %s", filepath) 

143 r = session.delete(filepath) 

144 if r.status_code not in [200, 202, 204]: 

145 raise FileNotFoundError(f"Unable to delete resource {filepath}; status code: {r.status_code}") 

146 

147 

148def folderExists(path: Union[Location, ButlerURI, str], 

149 session: Optional[requests.Session] = None) -> bool: 

150 """Check if the Webdav repository at a given URL actually exists. 

151 

152 Parameters 

153 ---------- 

154 path : `Location`, `ButlerURI` or `str` 

155 Location or ButlerURI containing the bucket name and filepath. 

156 session : `requests.Session`, optional 

157 Session object to query. 

158 

159 Returns 

160 ------- 

161 exists : `bool` 

162 True if it exists, False if no folder is found. 

163 """ 

164 if session is None: 

165 session = getHttpSession() 

166 

167 filepath = _getFileURL(path) 

168 

169 log.debug("Checking if folder exists: %s", filepath) 

170 r = session.head(filepath) 

171 return True if r.status_code == 200 else False 

172 

173 

174def isWebdavEndpoint(path: Union[Location, ButlerURI, str]) -> bool: 

175 """Check whether the remote HTTP endpoint implements Webdav features. 

176 

177 Parameters 

178 ---------- 

179 path : `Location`, `ButlerURI` or `str` 

180 Location or ButlerURI containing the bucket name and filepath. 

181 

182 Returns 

183 ------- 

184 isWebdav : `bool` 

185 True if the endpoint implements Webdav, False if it doesn't. 

186 """ 

187 filepath = _getFileURL(path) 

188 

189 log.debug("Detecting HTTP endpoint type...") 

190 r = requests.options(filepath) 

191 return True if 'DAV' in r.headers else False 

192 

193 

194def _getFileURL(path: Union[Location, ButlerURI, str]) -> str: 

195 """Returns the absolute URL of the resource as a string. 

196 

197 Parameters 

198 ---------- 

199 path : `Location`, `ButlerURI` or `str` 

200 Location or ButlerURI containing the bucket name and filepath. 

201 

202 Returns 

203 ------- 

204 filepath : `str` 

205 The fully qualified URL of the resource. 

206 """ 

207 if isinstance(path, Location): 

208 filepath = path.uri.geturl() 

209 else: 

210 filepath = ButlerURI(path).geturl() 

211 return filepath