Coverage for python/lsst/daf/butler/core/_butlerUri/schemeless.py: 55%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

72 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23from __future__ import annotations 

24 

25import os 

26import urllib.parse 

27import os.path 

28import logging 

29 

30__all__ = ('ButlerSchemelessURI',) 

31 

32from pathlib import PurePath 

33 

34from typing import ( 

35 Optional, 

36 Tuple, 

37 Union, 

38) 

39 

40from .file import ButlerFileURI 

41from .utils import os2posix 

42from ._butlerUri import ButlerURI 

43 

44log = logging.getLogger(__name__) 

45 

46 

47class ButlerSchemelessURI(ButlerFileURI): 

48 """Scheme-less URI referring to the local file system.""" 

49 

50 _pathLib = PurePath 

51 _pathModule = os.path 

52 quotePaths = False 

53 

54 @property 

55 def ospath(self) -> str: 

56 """Path component of the URI localized to current OS.""" 

57 return self.path 

58 

59 def isabs(self) -> bool: 

60 """Indicate that the resource is fully specified. 

61 

62 For non-schemeless URIs this is always true. 

63 

64 Returns 

65 ------- 

66 isabs : `bool` 

67 `True` if the file is absolute, `False` otherwise. 

68 """ 

69 return os.path.isabs(self.ospath) 

70 

71 def abspath(self) -> ButlerURI: 

72 """Force a schemeless URI to a file URI. 

73 

74 This will include URI quoting of the path. 

75 

76 Returns 

77 ------- 

78 file : `ButlerFileURI` 

79 A new URI using file scheme. 

80 

81 Notes 

82 ----- 

83 The current working directory will be used to convert this scheme-less 

84 URI to an absolute path. 

85 """ 

86 # Convert this URI to a string so that any fragments will be 

87 # processed correctly by the ButlerURI constructor. We provide 

88 # the options that will force the code below in _fixupPathUri to 

89 # return a file URI from a scheme-less one. 

90 return ButlerURI(str(self), forceAbsolute=True, forceDirectory=self.isdir(), 

91 isTemporary=self.isTemporary) 

92 

93 def relative_to(self, other: ButlerURI) -> Optional[str]: 

94 """Return the relative path from this URI to the other URI. 

95 

96 Parameters 

97 ---------- 

98 other : `ButlerURI` 

99 URI to use to calculate the relative path. 

100 

101 Returns 

102 ------- 

103 subpath : `str` 

104 The sub path of this URI relative to the supplied other URI. 

105 Returns `None` if there is no parent child relationship. 

106 If this URI is a relative URI but the other is 

107 absolute, it is assumed to be in the parent completely unless it 

108 starts with ".." (in which case the path is combined and tested). 

109 If both URIs are relative, the relative paths are compared 

110 for commonality. 

111 

112 Notes 

113 ----- 

114 By definition a relative path will be relative to the enclosing 

115 absolute parent URI. It will be returned unchanged if it does not 

116 use a parent directory specification. 

117 """ 

118 # In some scenarios below a new derived child URI needs to be created 

119 # to convert from scheme-less to absolute URI. 

120 child = None 

121 

122 if not self.isabs() and not other.isabs(): 

123 # Both are schemeless relative. Use parent implementation 

124 # rather than trying to convert both to file: first since schemes 

125 # match. 

126 pass 

127 elif not self.isabs() and other.isabs(): 

128 # Append child to other. This can account for .. in child path. 

129 child = other.join(self.path) 

130 elif self.isabs() and not other.isabs(): 

131 # Finding common paths is not possible if the parent is 

132 # relative and the child is absolute. 

133 return None 

134 elif self.isabs() and other.isabs(): 

135 # Both are absolute so convert schemeless to file 

136 # if necessary. 

137 child = self.abspath() 

138 if not other.scheme: 

139 other = other.abspath() 

140 else: 

141 raise RuntimeError(f"Unexpected combination of {child}.relative_to({other}).") 

142 

143 if child is None: 

144 return super().relative_to(other) 

145 return child.relative_to(other) 

146 

147 @classmethod 

148 def _fixupPathUri(cls, parsed: urllib.parse.ParseResult, root: Optional[Union[str, ButlerURI]] = None, 

149 forceAbsolute: bool = False, 

150 forceDirectory: bool = False) -> Tuple[urllib.parse.ParseResult, bool]: 

151 """Fix up relative paths for local file system. 

152 

153 Parameters 

154 ---------- 

155 parsed : `~urllib.parse.ParseResult` 

156 The result from parsing a URI using `urllib.parse`. 

157 root : `str` or `ButlerURI`, optional 

158 Path to use as root when converting relative to absolute. 

159 If `None`, it will be the current working directory. This 

160 is a local file system path, or a file URI. 

161 forceAbsolute : `bool`, optional 

162 If `True`, scheme-less relative URI will be converted to an 

163 absolute path using a ``file`` scheme. If `False` scheme-less URI 

164 will remain scheme-less and will not be updated to ``file`` or 

165 absolute path. 

166 forceDirectory : `bool`, optional 

167 If `True` forces the URI to end with a separator, otherwise given 

168 URI is interpreted as is. 

169 

170 Returns 

171 ------- 

172 modified : `~urllib.parse.ParseResult` 

173 Update result if a URI is being handled. 

174 dirLike : `bool` 

175 `True` if given parsed URI has a trailing separator or 

176 forceDirectory is True. Otherwise `False`. 

177 

178 Notes 

179 ----- 

180 Relative paths are explicitly not supported by RFC8089 but `urllib` 

181 does accept URIs of the form ``file:relative/path.ext``. They need 

182 to be turned into absolute paths before they can be used. This is 

183 always done regardless of the ``forceAbsolute`` parameter. 

184 

185 Scheme-less paths are normalized and environment variables are 

186 expanded. 

187 """ 

188 # assume we are not dealing with a directory URI 

189 dirLike = False 

190 

191 # Replacement values for the URI 

192 replacements = {} 

193 

194 if root is None: 194 ↛ 196line 194 didn't jump to line 196, because the condition on line 194 was never false

195 root = os.path.abspath(os.path.curdir) 

196 elif isinstance(root, ButlerURI): 

197 if root.scheme and root.scheme != "file": 

198 raise ValueError(f"The override root must be a file URI not {root.scheme}") 

199 root = os.path.abspath(root.ospath) 

200 

201 # this is a local OS file path which can support tilde expansion. 

202 # we quoted it in the constructor so unquote here 

203 expandedPath = os.path.expanduser(urllib.parse.unquote(parsed.path)) 

204 

205 # We might also be receiving a path containing environment variables 

206 # so expand those here 

207 expandedPath = os.path.expandvars(expandedPath) 

208 

209 # Ensure that this becomes a file URI if it is already absolute 

210 if os.path.isabs(expandedPath): 210 ↛ 211line 210 didn't jump to line 211, because the condition on line 210 was never true

211 replacements["scheme"] = "file" 

212 # Keep in OS form for now to simplify later logic 

213 replacements["path"] = os.path.normpath(expandedPath) 

214 elif forceAbsolute: 

215 # This can stay in OS path form, do not change to file 

216 # scheme. 

217 replacements["path"] = os.path.normpath(os.path.join(root, expandedPath)) 

218 else: 

219 # No change needed for relative local path staying relative 

220 # except normalization 

221 replacements["path"] = os.path.normpath(expandedPath) 

222 # normalization of empty path returns "." so we are dirLike 

223 if expandedPath == "": 223 ↛ 224line 223 didn't jump to line 224, because the condition on line 223 was never true

224 dirLike = True 

225 

226 # normpath strips trailing "/" which makes it hard to keep 

227 # track of directory vs file when calling replaceFile 

228 

229 # For local file system we can explicitly check to see if this 

230 # really is a directory. The URI might point to a location that 

231 # does not exists yet but all that matters is if it is a directory 

232 # then we make sure use that fact. No need to do the check if 

233 # we are already being told. 

234 if not forceDirectory and os.path.isdir(replacements["path"]): 234 ↛ 235line 234 didn't jump to line 235, because the condition on line 234 was never true

235 forceDirectory = True 

236 

237 # add the trailing separator only if explicitly required or 

238 # if it was stripped by normpath. Acknowledge that trailing 

239 # separator exists. 

240 endsOnSep = expandedPath.endswith(os.sep) and not replacements["path"].endswith(os.sep) 

241 if (forceDirectory or endsOnSep or dirLike): 

242 dirLike = True 

243 if not replacements["path"].endswith(os.sep): 243 ↛ 246line 243 didn't jump to line 246, because the condition on line 243 was never false

244 replacements["path"] += os.sep 

245 

246 if "scheme" in replacements: 246 ↛ 249line 246 didn't jump to line 249, because the condition on line 246 was never true

247 # This is now meant to be a URI path so force to posix 

248 # and quote 

249 replacements["path"] = urllib.parse.quote(os2posix(replacements["path"])) 

250 

251 # ParseResult is a NamedTuple so _replace is standard API 

252 parsed = parsed._replace(**replacements) 

253 

254 # We do allow fragment but do not expect params or query to be 

255 # specified for schemeless 

256 if parsed.params or parsed.query: 256 ↛ 257line 256 didn't jump to line 257, because the condition on line 256 was never true

257 log.warning("Additional items unexpectedly encountered in schemeless URI: %s", parsed.geturl()) 

258 

259 return parsed, dirLike