Coverage for python/lsst/daf/butler/core/webdavutils.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("getHttpSession", "isWebdavEndpoint", "webdavCheckFileExists",
25 "folderExists", "webdavDeleteFile")
27import os
28import requests
29import logging
30from requests.structures import CaseInsensitiveDict
32from typing import (
33 Optional,
34 Tuple,
35 Union,
36)
38from .location import ButlerURI, Location
40log = logging.getLogger(__name__)
43def getHttpSession() -> requests.Session:
44 """Create a requests.Session pre-configured with environment variable data
46 Returns
47 -------
48 session : `requests.Session`
49 An http session used to execute requests.
51 Notes
52 -----
53 The WEBDAV_AUTH_METHOD must be set to obtain a session.
54 Depending on the chosen method, additional
55 environment variables are required:
57 X509: must set WEBDAV_PROXY_CERT
58 (path to proxy certificate used to authenticate requests)
60 TOKEN: must set WEBDAV_BEARER_TOKEN
61 (bearer token used to authenticate requests, as a single string)
63 NB: requests will read CA certificates in REQUESTS_CA_BUNDLE
64 It must be manually exported according to the system CA directory.
65 """
66 session = requests.Session()
67 log.debug("Creating new HTTP session...")
69 try:
70 env_auth_method = os.environ['WEBDAV_AUTH_METHOD']
71 except KeyError:
72 raise KeyError("Environment variable WEBDAV_AUTH_METHOD is not set, please use values X509 or TOKEN")
74 if env_auth_method == "X509":
75 try:
76 proxy_cert = os.environ['WEBDAV_PROXY_CERT']
77 except KeyError:
78 raise KeyError("Environment variable WEBDAV_PROXY_CERT is not set")
79 log.debug("... using x509 authentication.")
80 session.cert = (proxy_cert, proxy_cert)
81 elif env_auth_method == "TOKEN":
82 try:
83 bearer_token = os.environ['WEBDAV_BEARER_TOKEN']
84 except KeyError:
85 raise KeyError("Environment variable WEBDAV_BEARER_TOKEN is not set")
86 log.debug("... using bearer-token authentication.")
87 session.headers = CaseInsensitiveDict({'Authorization': 'Bearer ' + bearer_token})
88 else:
89 raise ValueError("Environment variable WEBDAV_AUTH_METHOD must be set to X509 or TOKEN")
91 log.debug("Session configured and ready.")
93 return session
96def webdavCheckFileExists(path: Union[Location, ButlerURI, str],
97 session: Optional[requests.Session] = None) -> Tuple[bool, int]:
98 """Check that a remote HTTP resource exists.
100 Parameters
101 ----------
102 path : `Location`, `ButlerURI` or `str`
103 Location or ButlerURI containing the bucket name and filepath.
104 session : `requests.Session`, optional
105 Session object to query.
107 Returns
108 -------
109 exists : `bool`
110 True if resource exists, False otherwise.
111 size : `int`
112 Size of the resource, if it exists, in bytes, otherwise -1
113 """
114 if session is None:
115 session = getHttpSession()
117 filepath = _getFileURL(path)
119 log.debug("Checking if file exists: %s", filepath)
121 r = session.head(filepath)
122 return (True, int(r.headers['Content-Length'])) if r.status_code == 200 else (False, -1)
125def webdavDeleteFile(path: Union[Location, ButlerURI, str],
126 session: Optional[requests.Session] = None) -> None:
127 """Remove a remote HTTP resource.
128 Raises a FileNotFoundError if the resource does not exist or on failure.
130 Parameters
131 ----------
132 path : `Location`, `ButlerURI` or `str`
133 Location or ButlerURI containing the bucket name and filepath.
134 session : `requests.Session`, optional
135 Session object to query.
136 """
137 if session is None:
138 session = getHttpSession()
140 filepath = _getFileURL(path)
142 log.debug("Removing file: %s", filepath)
143 r = session.delete(filepath)
144 if r.status_code not in [200, 202, 204]:
145 raise FileNotFoundError(f"Unable to delete resource {filepath}; status code: {r.status_code}")
148def folderExists(path: Union[Location, ButlerURI, str],
149 session: Optional[requests.Session] = None) -> bool:
150 """Check if the Webdav repository at a given URL actually exists.
152 Parameters
153 ----------
154 path : `Location`, `ButlerURI` or `str`
155 Location or ButlerURI containing the bucket name and filepath.
156 session : `requests.Session`, optional
157 Session object to query.
159 Returns
160 -------
161 exists : `bool`
162 True if it exists, False if no folder is found.
163 """
164 if session is None:
165 session = getHttpSession()
167 filepath = _getFileURL(path)
169 log.debug("Checking if folder exists: %s", filepath)
170 r = session.head(filepath)
171 return True if r.status_code == 200 else False
174def isWebdavEndpoint(path: Union[Location, ButlerURI, str]) -> bool:
175 """Check whether the remote HTTP endpoint implements Webdav features.
177 Parameters
178 ----------
179 path : `Location`, `ButlerURI` or `str`
180 Location or ButlerURI containing the bucket name and filepath.
182 Returns
183 -------
184 isWebdav : `bool`
185 True if the endpoint implements Webdav, False if it doesn't.
186 """
187 filepath = _getFileURL(path)
189 log.debug("Detecting HTTP endpoint type...")
190 r = requests.options(filepath)
191 return True if 'DAV' in r.headers else False
194def _getFileURL(path: Union[Location, ButlerURI, str]) -> str:
195 """Returns the absolute URL of the resource as a string.
197 Parameters
198 ----------
199 path : `Location`, `ButlerURI` or `str`
200 Location or ButlerURI containing the bucket name and filepath.
202 Returns
203 -------
204 filepath : `str`
205 The fully qualified URL of the resource.
206 """
207 if isinstance(path, Location):
208 filepath = path.uri.geturl()
209 else:
210 filepath = ButlerURI(path).geturl()
211 return filepath