Coverage for python/lsst/daf/persistence/storageInterface.py: 89%

71 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-23 02:30 -0700

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2017 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24from abc import ABCMeta, abstractmethod 

25 

26 

27class NoRepositroyAtRoot(RuntimeError): 

28 pass 

29 

30 

31class StorageInterface: 

32 """Defines the interface for a connection to a Storage location. 

33 

34 Parameters 

35 ---------- 

36 uri : string 

37 URI or path that is used as the storage location. 

38 create : bool 

39 If True The StorageInterface subclass should create a new 

40 repository at the root location. If False then a new repository 

41 will not be created. 

42 

43 Raises 

44 ------ 

45 NoRepositroyAtRoot 

46 If create is False and a repository does not exist at the root 

47 specified by uri then NoRepositroyAtRoot is raised. 

48 """ 

49 __metaclass__ = ABCMeta 

50 

51 def __init__(self, uri, create): 

52 """initialzer""" 

53 pass 

54 

55 @classmethod 

56 def _readFormatters(cls): 

57 """Getter for the container of read formatters of a StorageInterface subclass. 

58 

59 Returns 

60 ------- 

61 dict 

62 The read formatters container belonging to the class type. 

63 """ 

64 try: 

65 return cls._readFormattersDict 

66 except AttributeError: 

67 cls._readFormattersDict = {} 

68 return cls._readFormattersDict 

69 

70 @classmethod 

71 def _writeFormatters(cls): 

72 """Getter for the container of write formatters of a StorageInterface subclass. 

73 

74 Returns 

75 ------- 

76 dict 

77 The write formatters container belonging to the class type. 

78 """ 

79 try: 

80 return cls._writeFormattersDict 

81 except AttributeError: 

82 cls._writeFormattersDict = {} 

83 return cls._writeFormattersDict 

84 

85 @classmethod 

86 def getReadFormatter(cls, objType): 

87 """Search in the registered formatters for the objType read formatter. 

88 

89 Parameters 

90 ---------- 

91 objType : class type 

92 The type of class to find a formatter for. 

93 

94 Returns 

95 ------- 

96 formatter callable 

97 The formatter callable used to read the object from the storageInterface. 

98 """ 

99 return cls._readFormatters().get(objType, None) 

100 

101 @classmethod 

102 def getWriteFormatter(cls, objType): 

103 """Search in the registered formatters for the objType write formatter. 

104 

105 Parameters 

106 ---------- 

107 objType : class type 

108 The type of class to find a formatter for. 

109 

110 Returns 

111 ------- 

112 formatter callable 

113 The formatter callable used to write the object to the storageInterface. 

114 """ 

115 return cls._writeFormatters().get(objType, None) 

116 

117 @classmethod 

118 def registerFormatters(cls, formatable, readFormatter=None, writeFormatter=None): 

119 """Register read and/or write formatters for a storageInterface subclass 

120 

121 Parameters 

122 ---------- 

123 cls : StorageInterface subclass 

124 The type of StorageInterface the formatter is being registered for. 

125 formatable : class object 

126 The class object whose instances can be formatted by the formatter. 

127 readFormatter : a read formatter callable 

128 The formatter function that can be used by the StorageInterface instance to read the object from 

129 the storage. 

130 writeFormatter : a write formatter callable 

131 The formatter function that can be used by the StorageInterface instance to write the object to 

132 the storage. 

133 

134 Raises 

135 ------ 

136 RuntimeError 

137 For each object type and StorageInterface subclass the read and write formatters should only be 

138 registered once. If a second registration occurs for either a RuntimeError is raised. 

139 """ 

140 def register(formatable, formatter, formatters, storageInterface): 

141 if formatable in formatters: 141 ↛ 142line 141 didn't jump to line 142, because the condition on line 141 was never true

142 raise RuntimeError(("Registration of second formatter {} for formattable {} in " 

143 " storageInterface {}").format(formatter, formatable, storageInterface)) 

144 formatters[formatable] = formatter 

145 

146 if readFormatter: 146 ↛ 149line 146 didn't jump to line 149, because the condition on line 146 was never false

147 formatters = cls._readFormatters() 

148 register(formatable, readFormatter, formatters, cls) 

149 if writeFormatter: 149 ↛ exitline 149 didn't return from function 'registerFormatters', because the condition on line 149 was never false

150 formatters = cls._writeFormatters() 

151 register(formatable, writeFormatter, formatters, cls) 

152 

153 @abstractmethod 

154 def write(self, butlerLocation, obj): 

155 """Writes an object to a location and persistence format specified by ButlerLocation 

156 

157 Parameters 

158 ---------- 

159 butlerLocation : ButlerLocation 

160 The location & formatting for the object to be written. 

161 obj : object instance 

162 The object to be written. 

163 """ 

164 

165 @abstractmethod 

166 def read(self, butlerLocation): 

167 """Read from a butlerLocation. 

168 

169 Parameters 

170 ---------- 

171 butlerLocation : ButlerLocation 

172 The location & formatting for the object(s) to be read. 

173 

174 Returns 

175 ------- 

176 A list of objects as described by the butler location. One item for 

177 each location in butlerLocation.getLocations() 

178 """ 

179 

180 @abstractmethod 

181 def getLocalFile(self, path): 

182 """Get a handle to a local copy of the file, downloading it to a 

183 temporary if needed. 

184 

185 Parameters 

186 ---------- 

187 path : string 

188 A path to the the file in storage, relative to root. 

189 

190 Returns 

191 ------- 

192 A handle to a local copy of the file. If storage is remote it will be 

193 a temporary file. If storage is local it may be the original file or 

194 a temporary file. The file name can be gotten via the 'name' property 

195 of the returned object. 

196 """ 

197 

198 @abstractmethod 

199 def exists(self, location): 

200 """Check if location exists. 

201 

202 Parameters 

203 ---------- 

204 location : ButlerLocation or string 

205 A a string or a ButlerLocation that describes the location of an 

206 object in this storage. 

207 

208 Returns 

209 ------- 

210 bool 

211 True if exists, else False. 

212 """ 

213 

214 @abstractmethod 

215 def instanceSearch(self, path): 

216 """Search for the given path in this storage instance. 

217 

218 If the path contains an HDU indicator (a number in brackets before the 

219 dot, e.g. 'foo.fits[1]', this will be stripped when searching and so 

220 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 

221 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 

222 

223 Parameters 

224 ---------- 

225 path : string 

226 A filename (and optionally prefix path) to search for within root. 

227 

228 Returns 

229 ------- 

230 string or None 

231 The location that was found, or None if no location was found. 

232 """ 

233 

234 @classmethod 

235 @abstractmethod 

236 def search(cls, root, path): 

237 """Look for the given path in the current root. 

238 

239 Also supports searching for the path in Butler v1 repositories by 

240 following the Butler v1 _parent symlink 

241 

242 If the path contains an HDU indicator (a number in brackets, e.g. 

243 'foo.fits[1]', this will be stripped when searching and so 

244 will match filenames without the HDU indicator, e.g. 'foo.fits'. The 

245 path returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 

246 

247 Parameters 

248 ---------- 

249 root : string 

250 The path to the root directory. 

251 path : string 

252 The path to the file within the root directory. 

253 

254 Returns 

255 ------- 

256 string or None 

257 The location that was found, or None if no location was found. 

258 """ 

259 

260 @abstractmethod 

261 def copyFile(self, fromLocation, toLocation): 

262 """Copy a file from one location to another on the local filesystem. 

263 

264 Parameters 

265 ---------- 

266 fromLocation : string 

267 Path and name of existing file. 

268 toLocation : string 

269 Path and name of new file. 

270 

271 Returns 

272 ------- 

273 None 

274 """ 

275 

276 @abstractmethod 

277 def locationWithRoot(self, location): 

278 """Get the full path to the location. 

279 

280 Parameters 

281 ---------- 

282 location : string 

283 Path to a location within the repository relative to repository 

284 root. 

285 

286 Returns 

287 ------- 

288 string 

289 Absolute path to to the locaiton within the repository. 

290 """ 

291 

292 @classmethod 

293 @abstractmethod 

294 def getRepositoryCfg(cls, uri): 

295 """Get a persisted RepositoryCfg 

296 

297 Parameters 

298 ---------- 

299 uri : URI or path to a RepositoryCfg 

300 Description 

301 

302 Returns 

303 ------- 

304 A RepositoryCfg instance or None 

305 """ 

306 

307 @classmethod 

308 @abstractmethod 

309 def putRepositoryCfg(cls, cfg, loc=None): 

310 """Serialize a RepositoryCfg to a location. 

311 

312 When loc == cfg.root, the RepositoryCfg is to be written at the root 

313 location of the repository. In that case, root is not written, it is 

314 implicit in the location of the cfg. This allows the cfg to move from 

315 machine to machine without modification. 

316 

317 Parameters 

318 ---------- 

319 cfg : RepositoryCfg instance 

320 The RepositoryCfg to be serailized. 

321 loc : string, optional 

322 The URI location (can be relative path) to write the RepositoryCfg. 

323 If loc is None, the location will be read from the root parameter 

324 of loc. 

325 

326 Returns 

327 ------- 

328 None 

329 """ 

330 

331 @classmethod 

332 @abstractmethod 

333 def getMapperClass(cls, root): 

334 """Get the mapper class associated with a repository root. 

335 

336 Parameters 

337 ---------- 

338 root : string 

339 The location of a persisted RepositoryCfg is (new style repos). 

340 

341 Returns 

342 ------- 

343 A class object or a class instance, depending on the state of the 

344 mapper when the repository was created. 

345 """ 

346 

347 # Optional: Only needs to work if relative paths are sensical on this 

348 # storage type and for the case where fromPath and toPath are of the same 

349 # storage type. 

350 @classmethod 

351 def relativePath(cls, fromPath, toPath): 

352 """Get a relative path from a location to a location. 

353 

354 Parameters 

355 ---------- 

356 fromPath : string 

357 A path at which to start. It can be a relative path or an 

358 absolute path. 

359 toPath : string 

360 A target location. It can be a relative path or an absolute path. 

361 

362 Returns 

363 ------- 

364 string 

365 A relative path that describes the path from fromPath to toPath. 

366 """ 

367 return toPath 

368 

369 # Optional: Only needs to work if relative paths and absolute paths are 

370 # sensical on this storage type and for the case where fromPath and toPath 

371 # are of the same storage type. 

372 @classmethod 

373 def absolutePath(cls, fromPath, relativePath): 

374 """Get an absolute path for the path from fromUri to toUri 

375 

376 Parameters 

377 ---------- 

378 fromPath : the starting location 

379 A location at which to start. It can be a relative path or an 

380 absolute path. 

381 relativePath : the location relative to fromPath 

382 A relative path. 

383 

384 Returns 

385 ------- 

386 string 

387 Path that is an absolute path representation of fromPath + 

388 relativePath, if one exists. If relativePath is absolute or if 

389 fromPath is not related to relativePath then relativePath will be 

390 returned. 

391 """ 

392 return relativePath