Coverage for python/lsst/daf/persistence/storage.py: 24%

90 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-19 12:17 -0700

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2016 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24 

25import urllib.parse 

26from . import NoRepositroyAtRoot 

27 

28 

29class Storage: 

30 """Base class for storages""" 

31 

32 storages = {} 

33 

34 def __init__(self): 

35 self.repositoryCfgs = {} 

36 

37 @staticmethod 

38 def registerStorageClass(scheme, cls): 

39 """Register derived classes for lookup by URI scheme. 

40 

41 A scheme is a name that describes the form a resource at the beginning of a URI 

42 e.g. 'http' indicates HTML and related code, such as is found in http://www.lsst.org 

43 

44 The only currently supported schemes are: 

45 * 'file' where the portion of the URI after the // indicates an absolute locaiton on disk. 

46 for example: file:/my_repository_folder/ 

47 * '' (no scheme) where the entire string is a relative path on the local system 

48 for example "my_repository_folder" will indicate a folder in the current working directory with the 

49 same name. 

50 

51 See documentation for the urlparse python library for more information. 

52 

53 .. warning:: 

54 

55 Storage is 'wet paint' and very likely to change during factorization of Butler back end and 

56 storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is 

57 strongly discouraged. 

58 

59 Parameters 

60 ---------- 

61 scheme : str 

62 Name of the `scheme` the class is being registered for, which appears at the beginning of a URI. 

63 cls : class object 

64 A class object that should be used for a given scheme. 

65 """ 

66 if scheme in Storage.storages: 66 ↛ 67line 66 didn't jump to line 67, because the condition on line 66 was never true

67 raise RuntimeError("Scheme '%s' already registered:%s" % (scheme, Storage.storages[scheme])) 

68 Storage.storages[scheme] = cls 

69 

70 def getRepositoryCfg(self, uri): 

71 """Get a RepositoryCfg from a location specified by uri. 

72 

73 If a cfg is found then it is cached by the uri, so that multiple lookups 

74 are not performed on storages that might be remote. 

75 

76 RepositoryCfgs are not supposed to change once they are created so this 

77 should not lead to stale data. 

78 """ 

79 cfg = self.repositoryCfgs.get(uri, None) 

80 if cfg: 

81 return cfg 

82 parseRes = urllib.parse.urlparse(uri) 

83 if parseRes.scheme in Storage.storages: 

84 cfg = Storage.storages[parseRes.scheme].getRepositoryCfg(uri) 

85 if cfg: 

86 self.repositoryCfgs[uri] = cfg 

87 else: 

88 raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme) 

89 return cfg 

90 

91 @staticmethod 

92 def putRepositoryCfg(cfg, uri): 

93 """Write a RepositoryCfg object to a location described by uri""" 

94 ret = None 

95 parseRes = urllib.parse.urlparse(uri) 

96 if parseRes.scheme in Storage.storages: 

97 ret = Storage.storages[parseRes.scheme].putRepositoryCfg(cfg, uri) 

98 else: 

99 raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme) 

100 return ret 

101 

102 @staticmethod 

103 def getMapperClass(uri): 

104 """Get a mapper class cfg value from location described by uri. 

105 

106 Note that in legacy repositories the mapper may be specified by a file called _mapper at the uri 

107 location, and in newer repositories the mapper would be specified by a RepositoryCfg stored at the uri 

108 location. 

109 

110 .. warning:: 

111 

112 Storage is 'wet paint' and very likely to change during factorization of Butler back end and 

113 storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is 

114 strongly discouraged. 

115 

116 """ 

117 ret = None 

118 parseRes = urllib.parse.urlparse(uri) 

119 if parseRes.scheme in Storage.storages: 

120 ret = Storage.storages[parseRes.scheme].getMapperClass(uri) 

121 else: 

122 raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme) 

123 return ret 

124 

125 @staticmethod 

126 def makeFromURI(uri, create=True): 

127 '''Instantiate a StorageInterface sublcass from a URI. 

128 

129 .. warning:: 

130 

131 Storage is 'wet paint' and very likely to change during factorization of Butler back end and 

132 storage formats (DM-6225). Use of it in production code other than via the 'old butler' API is 

133 strongly discouraged. 

134 

135 Parameters 

136 ---------- 

137 uri : string 

138 The uri to the root location of a repository. 

139 create : bool, optional 

140 If True The StorageInterface subclass should create a new 

141 repository at the root location. If False then a new repository 

142 will not be created. 

143 

144 Returns 

145 ------- 

146 A Storage subclass instance, or if create is False and a repository 

147 does not exist at the root location then returns None. 

148 

149 Raises 

150 ------ 

151 RuntimeError 

152 When a StorageInterface subclass does not exist for the scheme 

153 indicated by the uri. 

154 ''' 

155 ret = None 

156 parseRes = urllib.parse.urlparse(uri) 

157 if parseRes.scheme in Storage.storages: 

158 theClass = Storage.storages[parseRes.scheme] 

159 try: 

160 ret = theClass(uri=uri, create=create) 

161 except NoRepositroyAtRoot: 

162 pass 

163 else: 

164 raise RuntimeError("No storage registered for scheme %s" % parseRes.scheme) 

165 return ret 

166 

167 @staticmethod 

168 def isPosix(uri): 

169 """Test if a URI is for a local filesystem storage. 

170 

171 This is mostly for backward compatibility; Butler V1 repositories were only ever on the local 

172 filesystem. They may exist but not have a RepositoryCfg class. This enables conditional checking for a 

173 V1 Repository. 

174 

175 This function treats 'file' and '' (no scheme) as posix storages, see 

176 the class docstring for more details. 

177 

178 Parameters 

179 ---------- 

180 uri : string 

181 URI to the root of a Repository. 

182 

183 Returns 

184 ------- 

185 Bool 

186 True if the URI is associated with a posix storage, else false. 

187 """ 

188 parseRes = urllib.parse.urlparse(uri) 

189 if parseRes.scheme in ('file', ''): 

190 return True 

191 return False 

192 

193 @staticmethod 

194 def relativePath(fromUri, toUri): 

195 """Get a relative path from a location to a location, if a relative path for these 2 locations exists. 

196 

197 Parameters 

198 ---------- 

199 fromPath : string 

200 A URI that describes a location at which to start. 

201 toPath : string 

202 A URI that describes a target location. 

203 

204 Returns 

205 ------- 

206 string 

207 A relative path that describes the path from fromUri to toUri, provided one exists. If a relative 

208 path between the two URIs does not exist then the entire toUri path is returned. 

209 """ 

210 fromUriParseRes = urllib.parse.urlparse(fromUri) 

211 toUriParseRes = urllib.parse.urlparse(toUri) 

212 if fromUriParseRes.scheme != toUriParseRes.scheme: 

213 return toUri 

214 storage = Storage.storages.get(fromUriParseRes.scheme, None) 

215 if not storage: 

216 return toUri 

217 return storage.relativePath(fromUri, toUri) 

218 

219 @staticmethod 

220 def absolutePath(fromUri, toUri): 

221 """Get an absolute path for the path from fromUri to toUri 

222 

223 Parameters 

224 ---------- 

225 fromUri : the starting location 

226 Description 

227 toUri : the location relative to fromUri 

228 Description 

229 

230 Returns 

231 ------- 

232 string 

233 URI that is absolutepath fromUri + toUri, if one exists. If toUri is absolute or if fromUri is not 

234 related to toUri (e.g. are of different storage types) then toUri will be returned. 

235 """ 

236 fromUriParseRes = urllib.parse.urlparse(fromUri) 

237 toUriParseRes = urllib.parse.urlparse(toUri) 

238 if fromUriParseRes.scheme != toUriParseRes.scheme: 

239 return toUri 

240 storage = Storage.storages.get(fromUriParseRes.scheme, None) 

241 if not storage: 

242 return toUri 

243 return storage.absolutePath(fromUri, toUri) 

244 

245 @staticmethod 

246 def search(uri, path): 

247 """Look for the given path in a storage root at URI; return None if it can't be found. 

248 

249 If the path contains an HDU indicator (a number in brackets before the 

250 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 

251 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 

252 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 

253 

254 

255 Parameters 

256 ---------- 

257 root : string 

258 URI to the the root location to search 

259 path : string 

260 A filename (and optionally prefix path) to search for within root. 

261 

262 Returns 

263 ------- 

264 string or None 

265 The location that was found, or None if no location was found. 

266 """ 

267 parseRes = urllib.parse.urlparse(uri) 

268 storage = Storage.storages.get(parseRes.scheme, None) 

269 if storage: 

270 return storage.search(uri, path) 

271 return None 

272 

273 @staticmethod 

274 def storageExists(uri): 

275 """Ask if a storage at the location described by uri exists 

276 

277 Parameters 

278 ---------- 

279 root : string 

280 URI to the the root location of the storage 

281 

282 Returns 

283 ------- 

284 bool 

285 True if the storage exists, false if not 

286 """ 

287 parseRes = urllib.parse.urlparse(uri) 

288 storage = Storage.storages.get(parseRes.scheme, None) 

289 if storage: 

290 return storage.storageExists(uri) 

291 return None