Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

#!/usr/bin/env python 

 

# 

# LSST Data Management System 

# Copyright 2016 LSST Corporation. 

# 

# This product includes software developed by the 

# LSST Project (http://www.lsst.org/). 

# 

# This program is free software: you can redistribute it and/or modify 

# it under the terms of the GNU General Public License as published by 

# the Free Software Foundation, either version 3 of the License, or 

# (at your option) any later version. 

# 

# This program is distributed in the hope that it will be useful, 

# but WITHOUT ANY WARRANTY; without even the implied warranty of 

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

# GNU General Public License for more details. 

# 

# You should have received a copy of the LSST License Statement and 

# the GNU General Public License along with this program. If not, 

# see <http://www.lsstcorp.org/LegalNotices/>. 

# 

 

# -*- python -*- 

 

import copy 

import yaml 

from . import iterify, doImport, Storage, ParentsMismatch 

from past.builtins import basestring 

 

 

class RepositoryCfg(yaml.YAMLObject): 

"""RepositoryCfg stores the configuration of a repository. Its contents are persisted to the repository 

when the repository is created in persistent storage. Thereafter the the RepositoryCfg should not change. 

 

Parameters 

---------- 

mapper : string 

The mapper associated with the repository. The string should be importable to a class object. 

mapperArgs : dict 

Arguments & values to pass to the mapper when initializing it. 

parents : list of URI 

URIs to the locaiton of the parent RepositoryCfgs of this repository. 

policy : dict 

Policy associated with this repository, overrides all other policy data (which may be loaded from 

policies in derived packages). 

deserializing : bool 

Butler internal use only. This flag is used to indicate to the init funciton that the repository class 

is being deserialized and should not perform certain operations that normally happen in other uses of 

init. 

""" 

yaml_tag = u"!RepositoryCfg_v1" 

 

def __init__(self, root, mapper, mapperArgs, parents, policy): 

self._root = root 

self._mapper = mapper 

self._mapperArgs = {} if mapperArgs is None else mapperArgs 

self._parents = [] 

self.addParents(iterify(parents)) 

self._policy = policy 

self.dirty = True # if dirty, the parameters have been changed since the cfg was read or written. 

 

@staticmethod 

def v1Constructor(loader, node): 

"""Constructor for 'version 1' of the serlized RepositoryCfg. 

 

If new parameters are added to RepositoryCfg they will have to be checked for in d; if they are there 

then their value should be used and if they are not there a default value must be used in place. 

 

In case the structure of the serialzed file must be changed in a way that invalidates some of the 

keys: 

1. Increment the version number (after _v1) in the yaml_tag of this class. 

2. Add a new constructor (similar to this one) to deserialze new serializations of this class. 

3. Registered the new constructor for the new version with yaml, the same way it is done at the bottom 

of this file. 

4. All constructors for the older version(s) of persisted RepositoryCfg must be changed to adapt 

the old keys to their new uses and create the current (new) version of a repository cfg, or raise a 

RuntimeError in the case that older versions of serialized RepositoryCfgs can not be adapted. 

There is an example of migrating from a fictitious v0 to v1 in tests/repositoryCfg.py 

""" 

d = loader.construct_mapping(node) 

cfg = RepositoryCfg(root=d['_root'], mapper=d['_mapper'], mapperArgs=d['_mapperArgs'], 

parents=[], policy=d.get('_policy', None)) 

# Where possible we mangle the parents so that they are relative to root, for example if the root and 

# the parents are both in the same PosixStorage. The parents are serialized in mangled form; when 

# deserializing the parents we do not re-mangle them. 

cfg._parents = d['_parents'] 

if cfg._parents is None: 

cfg._parents = [] 

cfg.dirty = False 

return cfg 

 

def __eq__(self, other): 

if not other: 

return False 

return self.root == other.root and \ 

self.mapper == other.mapper and \ 

self.mapperArgs == other.mapperArgs and \ 

self.parents == other.parents and \ 

self.policy == other.policy 

 

def __ne__(self, other): 

return not self.__eq__(other) 

 

def extend(self, other): 

"""Extend this RepositoryCfg with extendable values from the other RepositoryCfg. 

 

Currently the only extendable value is parents; see `extendParents` for more detials about extending 

the parents list. 

 

Parameters 

---------- 

other : RepositoryCfg 

A RepositoryCfg instance to update values from. 

 

Raises 

------ 

RuntimeError 

If non-extendable parameters do not match a RuntimeError will be raised. 

(If this RepositoryCfg's parents can not be extended with the parents of the other repository, 

extendParents will raise). 

""" 

if (self.root != other.root or 

self.mapper != other.mapper or 

self.mapperArgs != other.mapperArgs or 

self.policy != other.policy): 

raise RuntimeError("{} can not be extended with cfg:{}".format(self, other)) 

self.extendParents(other.parents) 

 

def _extendsParents(self, newParents): 

"""Query if a list of parents starts with the same list of parents as this RepositoryCfg's parents, 

with new parents at the end. 

 

Parameters 

---------- 

newParents : list of string and/or RepositoryCfg 

A list of parents that contains all the parents that would be in this RepositoryCfg. 

This must include parents that may already be in this RepositoryCfg's parents list. Paths must be 

in absolute form (not relative). 

 

Returns 

------- 

bool 

True if the beginning of the new list matches this RepositoryCfg's parents list, False if not. 

""" 

doesExtendParents = False 

return doesExtendParents 

 

def extendParents(self, newParents): 

"""Determine if a parents list matches our parents list, with extra items at the end. If a list of 

parents does not match but the mismatch is because of new parents at the end of the list, then they 

can be added to the cfg. 

 

Parameters 

---------- 

newParents : list of string 

A list of parents that contains all the parents that are to be recorded into this RepositoryCfg. 

This must include parents that may already be in this RepositoryCfg's parents list 

 

Raises 

------ 

ParentsListMismatch 

Description 

""" 

newParents = self._normalizeParents(self.root, newParents) 

doRaise = False 

if self._parents != newParents: 

if all(x == y for (x, y) in zip(self._parents, newParents)): 

if len(self._parents) < len(newParents): 

self._parents = newParents 

self.dirty = True 

elif len(self._parents) == len(newParents): 

pass 

else: 

doRaise = True 

else: 

doRaise = True 

if doRaise: 

raise ParentsMismatch(("The beginning of the passed-in parents list: {} does not match the " + 

"existing parents list in this RepositoryCfg: {}").format( 

newParents, self._parents)) 

 

@property 

def root(self): 

return self._root 

 

@root.setter 

def root(self, root): 

if root is not None and self._root is not None: 

raise RuntimeError("Explicity clear root (set to None) before changing the value of root.") 

self._root = root 

 

@property 

def mapper(self): 

return self._mapper 

 

@mapper.setter 

def mapper(self, mapper): 

if self._mapper is not None: 

raise RuntimeError("Should not set mapper over previous not-None value.") 

self.dirty = True 

self._mapper = mapper 

 

@property 

def mapperArgs(self): 

return self._mapperArgs 

 

@mapperArgs.setter 

def mapperArgs(self, newDict): 

self.dirty = True 

self._mapperArgs = newDict 

 

@property 

def parents(self): 

return self._denormalizeParents(self.root, self._parents) 

 

@staticmethod 

def _normalizeParents(root, newParents): 

"""Eliminate symlinks in newParents and get the relative path (if one exists) from root to each parent 

root. 

 

Parameters 

---------- 

newParents : list containing strings and RepoistoryCfg instances 

Same as in `addParents`. 

 

Returns 

------- 

list of strings and RepositoryCfg instances. 

Normalized list of parents 

""" 

newParents = iterify(newParents) 

for i in range(len(newParents)): 

if isinstance(newParents[i], RepositoryCfg): 

newParents[i] = copy.copy(newParents[i]) 

parentRoot = newParents[i].root 

newParents[i].root = None 

newParents[i].root = Storage.relativePath(root, parentRoot) 

else: 

newParents[i] = Storage.relativePath(root, newParents[i]) 

return newParents 

 

@staticmethod 

def _denormalizeParents(root, parents): 

def getAbs(root, parent): 

if isinstance(parent, RepositoryCfg): 

parentRoot = parent.root 

parent.root = None 

parent.root = Storage.absolutePath(root, parentRoot) 

else: 

parent = Storage.absolutePath(root, parent) 

return parent 

return [getAbs(root, parent) for parent in parents] 

 

def addParents(self, newParents): 

"""Add a parent or list of parents to this RepositoryCfg 

 

Parameters 

---------- 

newParents : string or RepositoryCfg instance, or list of these. 

If string, newParents should be a path or URI to the parent 

repository. If RepositoryCfg, newParents should be a RepositoryCfg 

that describes the parent repository in part or whole. 

""" 

newParents = self._normalizeParents(self.root, newParents) 

for newParent in newParents: 

if newParent not in self._parents: 

self.dirty = True 

self._parents.append(newParent) 

 

@property 

def policy(self): 

return self._policy 

 

@staticmethod 

def makeFromArgs(repositoryArgs): 

cfg = RepositoryCfg(root=repositoryArgs.root, 

mapper=repositoryArgs.mapper, 

mapperArgs=repositoryArgs.mapperArgs, 

parents=None, 

policy=repositoryArgs.policy) 

return cfg 

 

def matchesArgs(self, repositoryArgs): 

"""Checks that a repositoryArgs instance will work with this repositoryCfg. This is useful 

when loading an already-existing repository that has a persisted cfg, to ensure that the args that are 

passed into butler do not conflict with the persisted cfg.""" 

if repositoryArgs.root is not None and self._root != repositoryArgs.root: 

return False 

 

repoArgsMapper = repositoryArgs.mapper 

cfgMapper = self._mapper 

if isinstance(repoArgsMapper, basestring): 

repoArgsMapper = doImport(repoArgsMapper) 

if isinstance(cfgMapper, basestring): 

cfgMapper = doImport(cfgMapper) 

if repoArgsMapper is not None and repoArgsMapper != cfgMapper: 

return False 

# check mapperArgs for any keys in common and if their value does not match then return false. 

if self._mapperArgs is not None and repositoryArgs.mapperArgs is not None: 

for key in set(self._mapperArgs.keys()) & set(repositoryArgs.mapperArgs): 

if self._mapperArgs[key] != repositoryArgs.mapperArgs[key]: 

return False 

if repositoryArgs.policy and repositoryArgs.policy != self._policy: 

return False 

 

return True 

 

def __repr__(self): 

return "%s(root=%r, mapper=%r, mapperArgs=%r, parents=%s, policy=%s)" % ( 

self.__class__.__name__, 

self._root, 

self._mapper, 

self._mapperArgs, 

self._parents, 

self._policy) 

 

 

yaml.add_constructor(u"!RepositoryCfg_v1", RepositoryCfg.v1Constructor)