Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2015 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24import collections 

25import collections.abc 

26import copy 

27import os 

28import warnings 

29import yaml 

30 

31import lsst.pex.policy as pexPolicy 

32import lsst.utils 

33 

34from yaml.representer import Representer 

35yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

36 

37 

38# UserDict and yaml have defined metaclasses and Python 3 does not allow multiple 

39# inheritance of classes with distinct metaclasses. We therefore have to 

40# create a new baseclass that Policy can inherit from. 

41class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)): 

42 pass 

43 

44 

45class _PolicyBase(collections.UserDict, yaml.YAMLObject, metaclass=_PolicyMeta): 

46 pass 

47 

48 

49class Policy(_PolicyBase): 

50 """Policy implements a datatype that is used by Butler for configuration parameters. 

51 It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be 

52 initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next: 

53 Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation. 

54 That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either 

55 of these syntaxes may be used. 

56 

57 Storage formats supported: 

58 - yaml: read and write is supported. 

59 - pex policy: read is supported, although this is deprecated and will at some point be removed. 

60 """ 

61 

62 def __init__(self, other=None): 

63 """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways: 

64 other (string) Treated as a path to a policy file on disk. Must end with '.paf' or '.yaml'. 

65 other (Pex Policy) Initializes this Policy with the values in the passed-in Pex Policy. 

66 other (Policy) Copies the other Policy's values into this one. 

67 other (dict) Copies the values from the dict into this Policy. 

68 """ 

69 collections.UserDict.__init__(self) 

70 

71 if other is None: 

72 return 

73 

74 if isinstance(other, collections.abc.Mapping): 

75 self.update(other) 

76 elif isinstance(other, Policy): 

77 self.data = copy.deepcopy(other.data) 

78 elif isinstance(other, str): 

79 # if other is a string, assume it is a file path. 

80 self.__initFromFile(other) 

81 elif isinstance(other, pexPolicy.Policy): 

82 # if other is an instance of a Pex Policy, load it accordingly. 

83 self.__initFromPexPolicy(other) 

84 else: 

85 # if the policy specified by other could not be loaded raise a runtime error. 

86 raise RuntimeError("A Policy could not be loaded from other:%s" % other) 

87 

88 def ppprint(self): 

89 """helper function for debugging, prints a policy out in a readable way in the debugger. 

90 

91 use: pdb> print myPolicyObject.pprint() 

92 :return: a prettyprint formatted string representing the policy 

93 """ 

94 import pprint 

95 return pprint.pformat(self.data, indent=2, width=1) 

96 

97 def __repr__(self): 

98 return self.data.__repr__() 

99 

100 def __initFromFile(self, path): 

101 """Load a file from path. If path is a list, will pick one to use, according to order specified 

102 by extensionPreference. 

103 

104 :param path: string or list of strings, to a persisted policy file. 

105 :param extensionPreference: the order in which to try to open files. Will use the first one that 

106 succeeds. 

107 :return: 

108 """ 

109 policy = None 

110 if path.endswith('yaml'): 

111 self.__initFromYamlFile(path) 

112 elif path.endswith('paf'): 

113 policy = pexPolicy.Policy.createPolicy(path) 

114 self.__initFromPexPolicy(policy) 

115 else: 

116 raise RuntimeError("Unhandled policy file type:%s" % path) 

117 

118 def __initFromPexPolicy(self, pexPolicy): 

119 """Load values from a pex policy. 

120 

121 :param pexPolicy: 

122 :return: 

123 """ 

124 names = pexPolicy.names() 

125 names.sort() 

126 for name in names: 

127 if pexPolicy.getValueType(name) == pexPolicy.POLICY: 

128 if name in self: 

129 continue 

130 else: 

131 self[name] = {} 

132 else: 

133 if pexPolicy.isArray(name): 

134 self[name] = pexPolicy.getArray(name) 

135 else: 

136 self[name] = pexPolicy.get(name) 

137 return self 

138 

139 def __initFromYamlFile(self, path): 

140 """Opens a file at a given path and attempts to load it in from yaml. 

141 

142 :param path: 

143 :return: 

144 """ 

145 with open(path, 'r') as f: 

146 self.__initFromYaml(f) 

147 

148 def __initFromYaml(self, stream): 

149 """Loads a YAML policy from any readable stream that contains one. 

150 

151 :param stream: 

152 :return: 

153 """ 

154 # will raise yaml.YAMLError if there is an error loading the file. 

155 try: 

156 # PyYAML >=5.1 prefers a different loader 

157 loader = yaml.FullLoader 

158 except AttributeError: 

159 loader = yaml.Loader 

160 self.data = yaml.load(stream, Loader=loader) 

161 return self 

162 

163 def __getitem__(self, name): 

164 data = self.data 

165 for key in name.split('.'): 

166 if data is None: 

167 return None 

168 if key in data: 

169 data = data[key] 

170 else: 

171 return None 

172 if isinstance(data, collections.abc.Mapping): 

173 data = Policy(data) 

174 return data 

175 

176 def __setitem__(self, name, value): 

177 if isinstance(value, collections.abc.Mapping): 

178 keys = name.split('.') 

179 d = {} 

180 cur = d 

181 for key in keys[0:-1]: 

182 cur[key] = {} 

183 cur = cur[key] 

184 cur[keys[-1]] = value 

185 self.update(d) 

186 data = self.data 

187 keys = name.split('.') 

188 for key in keys[0:-1]: 

189 data = data.setdefault(key, {}) 

190 data[keys[-1]] = value 

191 

192 def __contains__(self, key): 

193 d = self.data 

194 keys = key.split('.') 

195 for k in keys[0:-1]: 

196 if k in d: 

197 d = d[k] 

198 else: 

199 return False 

200 return keys[-1] in d 

201 

202 @staticmethod 

203 def defaultPolicyFile(productName, fileName, relativePath=None): 

204 """Get the path to a default policy file. 

205 

206 Determines a directory for the product specified by productName. Then Concatenates 

207 productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path 

208 to the default Policy file 

209 

210 @param productName (string) The name of the product that the default policy is installed as part of 

211 @param fileName (string) The name of the policy file. Can also include a path to the file relative to 

212 the directory where the product is installed. 

213 @param relativePath (string) The relative path from the directior where the product is installed to 

214 the location where the file (or the path to the file) is found. If None 

215 (default), the fileName argument is relative to the installation 

216 directory. 

217 """ 

218 basePath = lsst.utils.getPackageDir(productName) 

219 if not basePath: 

220 raise RuntimeError("No product installed for productName: %s" % basePath) 

221 if relativePath is not None: 

222 basePath = os.path.join(basePath, relativePath) 

223 fullFilePath = os.path.join(basePath, fileName) 

224 return fullFilePath 

225 

226 def update(self, other): 

227 """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested 

228 dict entirely. 

229 

230 For example, for the given code: 

231 foo = {'a': {'b': 1}} 

232 foo.update({'a': {'c': 2}}) 

233 

234 If foo is a dict, then after the update foo == {'a': {'c': 2}} 

235 But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}} 

236 """ 

237 def doUpdate(d, u): 

238 for k, v in u.items(): 

239 if isinstance(d, collections.abc.Mapping): 

240 if isinstance(v, collections.abc.Mapping): 

241 r = doUpdate(d.get(k, {}), v) 

242 d[k] = r 

243 else: 

244 d[k] = u[k] 

245 else: 

246 d = {k: u[k]} 

247 return d 

248 doUpdate(self.data, other) 

249 

250 def merge(self, other): 

251 """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and 

252 values that already exist in self will NOT be overwritten. 

253 

254 :param other: 

255 :return: 

256 """ 

257 otherCopy = copy.deepcopy(other) 

258 otherCopy.update(self) 

259 self.data = otherCopy.data 

260 

261 def names(self, topLevelOnly=False): 

262 """Get the dot-delimited name of all the keys in the hierarchy. 

263 NOTE: this is different than the built-in method dict.keys, which will return only the first level 

264 keys. 

265 """ 

266 if topLevelOnly: 

267 return list(self.keys()) 

268 

269 def getKeys(d, keys, base): 

270 for key in d: 

271 val = d[key] 

272 levelKey = base + '.' + key if base is not None else key 

273 keys.append(levelKey) 

274 if isinstance(val, collections.abc.Mapping): 

275 getKeys(val, keys, levelKey) 

276 keys = [] 

277 getKeys(self.data, keys, None) 

278 return keys 

279 

280 def asArray(self, name): 

281 """Get a value as an array. May contain one or more elements. 

282 

283 :param key: 

284 :return: 

285 """ 

286 val = self.get(name) 

287 if isinstance(val, str): 

288 val = [val] 

289 elif not isinstance(val, collections.abc.Container): 

290 val = [val] 

291 return val 

292 

293 # Deprecated methods that mimic pex_policy api. 

294 # These are supported (for now), but callers should use the dict api. 

295 

296 def getValue(self, name): 

297 """Get the value for a parameter name/key. See class notes about dot-delimited access. 

298 

299 :param name: 

300 :return: the value for the given name. 

301 """ 

302 warnings.warn_explicit("Deprecated. Use []", DeprecationWarning) 

303 return self[name] 

304 

305 def setValue(self, name, value): 

306 """Set the value for a parameter name/key. See class notes about dot-delimited access. 

307 

308 :param name: 

309 :return: None 

310 """ 

311 warnings.warn("Deprecated. Use []", DeprecationWarning) 

312 self[name] = value 

313 

314 def mergeDefaults(self, other): 

315 """For any keys in other that are not present in self, sets that key and its value into self. 

316 

317 :param other: another Policy 

318 :return: None 

319 """ 

320 warnings.warn("Deprecated. Use .merge()", DeprecationWarning) 

321 self.merge(other) 

322 

323 def exists(self, key): 

324 """Query if a key exists in this Policy 

325 

326 :param key: 

327 :return: True if the key exists, else false. 

328 """ 

329 warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning) 

330 return key in self 

331 

332 def getString(self, key): 

333 """Get the string value of a key. 

334 

335 :param key: 

336 :return: the value for key 

337 """ 

338 warnings.warn("Deprecated. Use []", DeprecationWarning) 

339 return str(self[key]) 

340 

341 def getBool(self, key): 

342 """Get the value of a key. 

343 

344 :param key: 

345 :return: the value for key 

346 """ 

347 warnings.warn("Deprecated. Use []", DeprecationWarning) 

348 return bool(self[key]) 

349 

350 def getPolicy(self, key): 

351 """Get a subpolicy. 

352 

353 :param key: 

354 :return: 

355 """ 

356 warnings.warn("Deprecated. Use []", DeprecationWarning) 

357 return self[key] 

358 

359 def getStringArray(self, key): 

360 """Get a value as an array. May contain one or more elements. 

361 

362 :param key: 

363 :return: 

364 """ 

365 warnings.warn("Deprecated. Use asArray()", DeprecationWarning) 

366 val = self.get(key) 

367 if isinstance(val, str): 

368 val = [val] 

369 elif not isinstance(val, collections.abc.Container): 

370 val = [val] 

371 return val 

372 

373 def __lt__(self, other): 

374 if isinstance(other, Policy): 

375 other = other.data 

376 return self.data < other 

377 

378 def __le__(self, other): 

379 if isinstance(other, Policy): 

380 other = other.data 

381 return self.data <= other 

382 

383 def __eq__(self, other): 

384 if isinstance(other, Policy): 

385 other = other.data 

386 return self.data == other 

387 

388 def __ne__(self, other): 

389 if isinstance(other, Policy): 

390 other = other.data 

391 return self.data != other 

392 

393 def __gt__(self, other): 

394 if isinstance(other, Policy): 

395 other = other.data 

396 return self.data > other 

397 

398 def __ge__(self, other): 

399 if isinstance(other, Policy): 

400 other = other.data 

401 return self.data >= other 

402 

403 ####### 

404 # i/o # 

405 

406 def dump(self, output): 

407 """Writes the policy to a yaml stream. 

408 

409 :param stream: 

410 :return: 

411 """ 

412 # First a set of known keys is handled and written to the stream in a specific order for readability. 

413 # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to 

414 # the stream. 

415 data = copy.copy(self.data) 

416 keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera', 

417 'exposures', 'calibrations', 'datasets'] 

418 for key in keys: 

419 try: 

420 yaml.safe_dump({key: data.pop(key)}, output, default_flow_style=False) 

421 output.write('\n') 

422 except KeyError: 

423 pass 

424 if data: 

425 yaml.safe_dump(data, output, default_flow_style=False) 

426 

427 def dumpToFile(self, path): 

428 """Writes the policy to a file. 

429 

430 :param path: 

431 :return: 

432 """ 

433 with open(path, 'w') as f: 

434 self.dump(f)