Coverage for python/lsst/daf/persistence/repositoryCfg.py: 24%

137 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-06 12:53 -0800

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2016 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24 

25# -*- python -*- 

26 

27import copy 

28import yaml 

29from . import iterify, doImport, Storage, ParentsMismatch 

30 

31 

32class RepositoryCfg(yaml.YAMLObject): 

33 """RepositoryCfg stores the configuration of a repository. Its contents are persisted to the repository 

34 when the repository is created in persistent storage. Thereafter the the RepositoryCfg should not change. 

35 

36 Parameters 

37 ---------- 

38 mapper : string 

39 The mapper associated with the repository. The string should be importable to a class object. 

40 mapperArgs : dict 

41 Arguments & values to pass to the mapper when initializing it. 

42 parents : list of URI 

43 URIs to the locaiton of the parent RepositoryCfgs of this repository. 

44 policy : dict 

45 Policy associated with this repository, overrides all other policy data (which may be loaded from 

46 policies in derived packages). 

47 deserializing : bool 

48 Butler internal use only. This flag is used to indicate to the init funciton that the repository class 

49 is being deserialized and should not perform certain operations that normally happen in other uses of 

50 init. 

51 """ 

52 yaml_tag = u"!RepositoryCfg_v1" 

53 

54 def __init__(self, root, mapper, mapperArgs, parents, policy): 

55 self._root = root 

56 self._mapper = mapper 

57 self._mapperArgs = {} if mapperArgs is None else mapperArgs 

58 self._parents = [] 

59 self.addParents(iterify(parents)) 

60 self._policy = policy 

61 self.dirty = True # if dirty, the parameters have been changed since the cfg was read or written. 

62 

63 @staticmethod 

64 def v1Constructor(loader, node): 

65 """Constructor for 'version 1' of the serlized RepositoryCfg. 

66 

67 If new parameters are added to RepositoryCfg they will have to be checked for in d; if they are there 

68 then their value should be used and if they are not there a default value must be used in place. 

69 

70 In case the structure of the serialzed file must be changed in a way that invalidates some of the 

71 keys: 

72 1. Increment the version number (after _v1) in the yaml_tag of this class. 

73 2. Add a new constructor (similar to this one) to deserialze new serializations of this class. 

74 3. Registered the new constructor for the new version with yaml, the same way it is done at the bottom 

75 of this file. 

76 4. All constructors for the older version(s) of persisted RepositoryCfg must be changed to adapt 

77 the old keys to their new uses and create the current (new) version of a repository cfg, or raise a 

78 RuntimeError in the case that older versions of serialized RepositoryCfgs can not be adapted. 

79 There is an example of migrating from a fictitious v0 to v1 in tests/repositoryCfg.py 

80 """ 

81 d = loader.construct_mapping(node) 

82 cfg = RepositoryCfg(root=d['_root'], mapper=d['_mapper'], mapperArgs=d['_mapperArgs'], 

83 parents=[], policy=d.get('_policy', None)) 

84 # Where possible we mangle the parents so that they are relative to root, for example if the root and 

85 # the parents are both in the same PosixStorage. The parents are serialized in mangled form; when 

86 # deserializing the parents we do not re-mangle them. 

87 cfg._parents = d['_parents'] 

88 if cfg._parents is None: 

89 cfg._parents = [] 

90 cfg.dirty = False 

91 return cfg 

92 

93 def __eq__(self, other): 

94 if not other: 

95 return False 

96 return self.root == other.root and \ 

97 self.mapper == other.mapper and \ 

98 self.mapperArgs == other.mapperArgs and \ 

99 self.parents == other.parents and \ 

100 self.policy == other.policy 

101 

102 def __ne__(self, other): 

103 return not self.__eq__(other) 

104 

105 def extend(self, other): 

106 """Extend this RepositoryCfg with extendable values from the other RepositoryCfg. 

107 

108 Currently the only extendable value is parents; see `extendParents` for more detials about extending 

109 the parents list. 

110 

111 Parameters 

112 ---------- 

113 other : RepositoryCfg 

114 A RepositoryCfg instance to update values from. 

115 

116 Raises 

117 ------ 

118 RuntimeError 

119 If non-extendable parameters do not match a RuntimeError will be raised. 

120 (If this RepositoryCfg's parents can not be extended with the parents of the other repository, 

121 extendParents will raise). 

122 """ 

123 if (self.root != other.root 

124 or self.mapper != other.mapper 

125 or self.mapperArgs != other.mapperArgs 

126 or self.policy != other.policy): 

127 raise RuntimeError("{} can not be extended with cfg:{}".format(self, other)) 

128 self.extendParents(other.parents) 

129 

130 def _extendsParents(self, newParents): 

131 """Query if a list of parents starts with the same list of parents as this RepositoryCfg's parents, 

132 with new parents at the end. 

133 

134 Parameters 

135 ---------- 

136 newParents : list of string and/or RepositoryCfg 

137 A list of parents that contains all the parents that would be in this RepositoryCfg. 

138 This must include parents that may already be in this RepositoryCfg's parents list. Paths must be 

139 in absolute form (not relative). 

140 

141 Returns 

142 ------- 

143 bool 

144 True if the beginning of the new list matches this RepositoryCfg's parents list, False if not. 

145 """ 

146 doesExtendParents = False 

147 return doesExtendParents 

148 

149 def extendParents(self, newParents): 

150 """Determine if a parents list matches our parents list, with extra items at the end. If a list of 

151 parents does not match but the mismatch is because of new parents at the end of the list, then they 

152 can be added to the cfg. 

153 

154 Parameters 

155 ---------- 

156 newParents : list of string 

157 A list of parents that contains all the parents that are to be recorded into this RepositoryCfg. 

158 This must include parents that may already be in this RepositoryCfg's parents list 

159 

160 Raises 

161 ------ 

162 ParentsListMismatch 

163 Description 

164 """ 

165 newParents = self._normalizeParents(self.root, newParents) 

166 doRaise = False 

167 if self._parents != newParents: 

168 if all(x == y for (x, y) in zip(self._parents, newParents)): 

169 if len(self._parents) < len(newParents): 

170 self._parents = newParents 

171 self.dirty = True 

172 elif len(self._parents) == len(newParents): 

173 pass 

174 else: 

175 doRaise = True 

176 else: 

177 doRaise = True 

178 if doRaise: 

179 raise ParentsMismatch(("The beginning of the passed-in parents list: {} does not match the " 

180 "existing parents list in this RepositoryCfg: {}").format( 

181 newParents, self._parents)) 

182 

183 @property 

184 def root(self): 

185 return self._root 

186 

187 @root.setter 

188 def root(self, root): 

189 if root is not None and self._root is not None: 

190 raise RuntimeError("Explicity clear root (set to None) before changing the value of root.") 

191 self._root = root 

192 

193 @property 

194 def mapper(self): 

195 return self._mapper 

196 

197 @mapper.setter 

198 def mapper(self, mapper): 

199 if self._mapper is not None: 

200 raise RuntimeError("Should not set mapper over previous not-None value.") 

201 self.dirty = True 

202 self._mapper = mapper 

203 

204 @property 

205 def mapperArgs(self): 

206 return self._mapperArgs 

207 

208 @mapperArgs.setter 

209 def mapperArgs(self, newDict): 

210 self.dirty = True 

211 self._mapperArgs = newDict 

212 

213 @property 

214 def parents(self): 

215 return self._denormalizeParents(self.root, self._parents) 

216 

217 @staticmethod 

218 def _normalizeParents(root, newParents): 

219 """Eliminate symlinks in newParents and get the relative path (if one exists) from root to each parent 

220 root. 

221 

222 Parameters 

223 ---------- 

224 newParents : list containing strings and RepoistoryCfg instances 

225 Same as in `addParents`. 

226 

227 Returns 

228 ------- 

229 list of strings and RepositoryCfg instances. 

230 Normalized list of parents 

231 """ 

232 newParents = iterify(newParents) 

233 for i in range(len(newParents)): 

234 if isinstance(newParents[i], RepositoryCfg): 

235 newParents[i] = copy.copy(newParents[i]) 

236 parentRoot = newParents[i].root 

237 newParents[i].root = None 

238 newParents[i].root = Storage.relativePath(root, parentRoot) 

239 else: 

240 newParents[i] = Storage.relativePath(root, newParents[i]) 

241 return newParents 

242 

243 @staticmethod 

244 def _denormalizeParents(root, parents): 

245 def getAbs(root, parent): 

246 if isinstance(parent, RepositoryCfg): 

247 parentRoot = parent.root 

248 parent.root = None 

249 parent.root = Storage.absolutePath(root, parentRoot) 

250 else: 

251 parent = Storage.absolutePath(root, parent) 

252 return parent 

253 return [getAbs(root, parent) for parent in parents] 

254 

255 def addParents(self, newParents): 

256 """Add a parent or list of parents to this RepositoryCfg 

257 

258 Parameters 

259 ---------- 

260 newParents : string or RepositoryCfg instance, or list of these. 

261 If string, newParents should be a path or URI to the parent 

262 repository. If RepositoryCfg, newParents should be a RepositoryCfg 

263 that describes the parent repository in part or whole. 

264 """ 

265 newParents = self._normalizeParents(self.root, newParents) 

266 for newParent in newParents: 

267 if newParent not in self._parents: 

268 self.dirty = True 

269 self._parents.append(newParent) 

270 

271 @property 

272 def policy(self): 

273 return self._policy 

274 

275 @staticmethod 

276 def makeFromArgs(repositoryArgs): 

277 cfg = RepositoryCfg(root=repositoryArgs.root, 

278 mapper=repositoryArgs.mapper, 

279 mapperArgs=repositoryArgs.mapperArgs, 

280 parents=None, 

281 policy=repositoryArgs.policy) 

282 return cfg 

283 

284 def matchesArgs(self, repositoryArgs): 

285 """Checks that a repositoryArgs instance will work with this repositoryCfg. This is useful 

286 when loading an already-existing repository that has a persisted cfg, to ensure that the args that are 

287 passed into butler do not conflict with the persisted cfg.""" 

288 if repositoryArgs.root is not None and self._root != repositoryArgs.root: 

289 return False 

290 

291 repoArgsMapper = repositoryArgs.mapper 

292 cfgMapper = self._mapper 

293 if isinstance(repoArgsMapper, str): 

294 repoArgsMapper = doImport(repoArgsMapper) 

295 if isinstance(cfgMapper, str): 

296 cfgMapper = doImport(cfgMapper) 

297 if repoArgsMapper is not None and repoArgsMapper != cfgMapper: 

298 return False 

299 # check mapperArgs for any keys in common and if their value does not match then return false. 

300 if self._mapperArgs is not None and repositoryArgs.mapperArgs is not None: 

301 for key in set(self._mapperArgs.keys()) & set(repositoryArgs.mapperArgs): 

302 if self._mapperArgs[key] != repositoryArgs.mapperArgs[key]: 

303 return False 

304 if repositoryArgs.policy and repositoryArgs.policy != self._policy: 

305 return False 

306 

307 return True 

308 

309 def __repr__(self): 

310 return "%s(root=%r, mapper=%r, mapperArgs=%r, parents=%s, policy=%s)" % ( 

311 self.__class__.__name__, 

312 self._root, 

313 self._mapper, 

314 self._mapperArgs, 

315 self._parents, 

316 self._policy) 

317 

318 

319loaderList = [yaml.Loader, ] 

320try: 

321 loaderList.append(yaml.UnsafeLoader) 

322except AttributeError: 

323 pass 

324 

325for loader in loaderList: 

326 yaml.add_constructor(u"!RepositoryCfg_v1", RepositoryCfg.v1Constructor, Loader=loader)