Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23from __future__ import annotations 

24 

25import os 

26import urllib.parse 

27import os.path 

28import logging 

29 

30__all__ = ('ButlerSchemelessURI',) 

31 

32from pathlib import PurePath 

33 

34from typing import ( 

35 Optional, 

36 Tuple, 

37 Union, 

38) 

39 

40from .file import ButlerFileURI 

41from .utils import os2posix 

42from ._butlerUri import ButlerURI 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class ButlerSchemelessURI(ButlerFileURI): 

48 """Scheme-less URI referring to the local file system.""" 

49 

50 _pathLib = PurePath 

51 _pathModule = os.path 

52 quotePaths = False 

53 

54 @property 

55 def ospath(self) -> str: 

56 """Path component of the URI localized to current OS.""" 

57 return self.path 

58 

59 def isabs(self) -> bool: 

60 """Indicate that the resource is fully specified. 

61 

62 For non-schemeless URIs this is always true. 

63 

64 Returns 

65 ------- 

66 isabs : `bool` 

67 `True` if the file is absolute, `False` otherwise. 

68 """ 

69 return os.path.isabs(self.ospath) 

70 

71 def _force_to_file(self) -> ButlerFileURI: 

72 """Force a schemeless URI to a file URI and returns a new URI. 

73 

74 This will include URI quoting of the path. 

75 

76 Returns 

77 ------- 

78 file : `ButlerFileURI` 

79 A copy of the URI using file scheme. If already a file scheme 

80 the copy will be identical. 

81 

82 Raises 

83 ------ 

84 ValueError 

85 Raised if this URI is schemeless and relative path and so can 

86 not be forced to file absolute path without context. 

87 """ 

88 if not self.isabs(): 

89 raise RuntimeError(f"Internal error: Can not force {self} to absolute file URI") 

90 uri = self._uri._replace(scheme="file", path=urllib.parse.quote(os2posix(self.path))) 

91 # mypy really wants a ButlerFileURI to be returned here 

92 return ButlerURI(uri, forceDirectory=self.dirLike) # type: ignore 

93 

94 @classmethod 

95 def _fixupPathUri(cls, parsed: urllib.parse.ParseResult, root: Optional[Union[str, ButlerURI]] = None, 

96 forceAbsolute: bool = False, 

97 forceDirectory: bool = False) -> Tuple[urllib.parse.ParseResult, bool]: 

98 """Fix up relative paths for local file system. 

99 

100 Parameters 

101 ---------- 

102 parsed : `~urllib.parse.ParseResult` 

103 The result from parsing a URI using `urllib.parse`. 

104 root : `str` or `ButlerURI`, optional 

105 Path to use as root when converting relative to absolute. 

106 If `None`, it will be the current working directory. This 

107 is a local file system path, or a file URI. 

108 forceAbsolute : `bool`, optional 

109 If `True`, scheme-less relative URI will be converted to an 

110 absolute path using a ``file`` scheme. If `False` scheme-less URI 

111 will remain scheme-less and will not be updated to ``file`` or 

112 absolute path. 

113 forceDirectory : `bool`, optional 

114 If `True` forces the URI to end with a separator, otherwise given 

115 URI is interpreted as is. 

116 

117 Returns 

118 ------- 

119 modified : `~urllib.parse.ParseResult` 

120 Update result if a URI is being handled. 

121 dirLike : `bool` 

122 `True` if given parsed URI has a trailing separator or 

123 forceDirectory is True. Otherwise `False`. 

124 

125 Notes 

126 ----- 

127 Relative paths are explicitly not supported by RFC8089 but `urllib` 

128 does accept URIs of the form ``file:relative/path.ext``. They need 

129 to be turned into absolute paths before they can be used. This is 

130 always done regardless of the ``forceAbsolute`` parameter. 

131 

132 Scheme-less paths are normalized and environment variables are 

133 expanded. 

134 """ 

135 # assume we are not dealing with a directory URI 

136 dirLike = False 

137 

138 # Replacement values for the URI 

139 replacements = {} 

140 

141 if root is None: 141 ↛ 143line 141 didn't jump to line 143, because the condition on line 141 was never false

142 root = os.path.abspath(os.path.curdir) 

143 elif isinstance(root, ButlerURI): 

144 if root.scheme and root.scheme != "file": 

145 raise RuntimeError(f"The override root must be a file URI not {root.scheme}") 

146 root = os.path.abspath(root.ospath) 

147 

148 # this is a local OS file path which can support tilde expansion. 

149 # we quoted it in the constructor so unquote here 

150 expandedPath = os.path.expanduser(urllib.parse.unquote(parsed.path)) 

151 

152 # We might also be receiving a path containing environment variables 

153 # so expand those here 

154 expandedPath = os.path.expandvars(expandedPath) 

155 

156 # Ensure that this becomes a file URI if it is already absolute 

157 if os.path.isabs(expandedPath): 157 ↛ 158line 157 didn't jump to line 158, because the condition on line 157 was never true

158 replacements["scheme"] = "file" 

159 # Keep in OS form for now to simplify later logic 

160 replacements["path"] = os.path.normpath(expandedPath) 

161 elif forceAbsolute: 

162 # This can stay in OS path form, do not change to file 

163 # scheme. 

164 replacements["path"] = os.path.normpath(os.path.join(root, expandedPath)) 

165 else: 

166 # No change needed for relative local path staying relative 

167 # except normalization 

168 replacements["path"] = os.path.normpath(expandedPath) 

169 # normalization of empty path returns "." so we are dirLike 

170 if expandedPath == "": 170 ↛ 171line 170 didn't jump to line 171, because the condition on line 170 was never true

171 dirLike = True 

172 

173 # normpath strips trailing "/" which makes it hard to keep 

174 # track of directory vs file when calling replaceFile 

175 

176 # For local file system we can explicitly check to see if this 

177 # really is a directory. The URI might point to a location that 

178 # does not exists yet but all that matters is if it is a directory 

179 # then we make sure use that fact. No need to do the check if 

180 # we are already being told. 

181 if not forceDirectory and os.path.isdir(replacements["path"]): 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true

182 forceDirectory = True 

183 

184 # add the trailing separator only if explicitly required or 

185 # if it was stripped by normpath. Acknowledge that trailing 

186 # separator exists. 

187 endsOnSep = expandedPath.endswith(os.sep) and not replacements["path"].endswith(os.sep) 

188 if (forceDirectory or endsOnSep or dirLike): 

189 dirLike = True 

190 if not replacements["path"].endswith(os.sep): 190 ↛ 193line 190 didn't jump to line 193, because the condition on line 190 was never false

191 replacements["path"] += os.sep 

192 

193 if "scheme" in replacements: 193 ↛ 196line 193 didn't jump to line 196, because the condition on line 193 was never true

194 # This is now meant to be a URI path so force to posix 

195 # and quote 

196 replacements["path"] = urllib.parse.quote(os2posix(replacements["path"])) 

197 

198 # ParseResult is a NamedTuple so _replace is standard API 

199 parsed = parsed._replace(**replacements) 

200 

201 # We do allow fragment but do not expect params or query to be 

202 # specified for schemeless 

203 if parsed.params or parsed.query: 203 ↛ 204line 203 didn't jump to line 204, because the condition on line 203 was never true

204 log.warning("Additional items unexpectedly encountered in schemeless URI: %s", parsed.geturl()) 

205 

206 return parsed, dirLike