Coverage for python/lsst/daf/persistence/policy.py: 17%

189 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-10-12 02:38 -0700

1#!/usr/bin/env python 

2 

3# 

4# LSST Data Management System 

5# Copyright 2015 LSST Corporation. 

6# 

7# This product includes software developed by the 

8# LSST Project (http://www.lsst.org/). 

9# 

10# This program is free software: you can redistribute it and/or modify 

11# it under the terms of the GNU General Public License as published by 

12# the Free Software Foundation, either version 3 of the License, or 

13# (at your option) any later version. 

14# 

15# This program is distributed in the hope that it will be useful, 

16# but WITHOUT ANY WARRANTY; without even the implied warranty of 

17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

18# GNU General Public License for more details. 

19# 

20# You should have received a copy of the LSST License Statement and 

21# the GNU General Public License along with this program. If not, 

22# see <http://www.lsstcorp.org/LegalNotices/>. 

23# 

24import collections 

25import collections.abc 

26import copy 

27import os 

28import warnings 

29import yaml 

30 

31import lsst.utils 

32 

33from yaml.representer import Representer 

34yaml.add_representer(collections.defaultdict, Representer.represent_dict) 

35 

36 

37# UserDict and yaml have defined metaclasses and Python 3 does not allow multiple 

38# inheritance of classes with distinct metaclasses. We therefore have to 

39# create a new baseclass that Policy can inherit from. 

40class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)): 

41 pass 

42 

43 

44class _PolicyBase(collections.UserDict, yaml.YAMLObject, metaclass=_PolicyMeta): 

45 pass 

46 

47 

48class Policy(_PolicyBase): 

49 """Policy implements a datatype that is used by Butler for configuration parameters. 

50 It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be 

51 initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next: 

52 Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation. 

53 That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either 

54 of these syntaxes may be used. 

55 

56 Storage format supported: 

57 - yaml: read and write is supported. 

58 """ 

59 

60 def __init__(self, other=None): 

61 """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways: 

62 other (string) Treated as a path to a policy file on disk. Must end with '.yaml'. 

63 other (Policy) Copies the other Policy's values into this one. 

64 other (dict) Copies the values from the dict into this Policy. 

65 """ 

66 collections.UserDict.__init__(self) 

67 

68 if other is None: 

69 return 

70 

71 if isinstance(other, collections.abc.Mapping): 

72 self.update(other) 

73 elif isinstance(other, Policy): 

74 self.data = copy.deepcopy(other.data) 

75 elif isinstance(other, str): 

76 # if other is a string, assume it is a file path. 

77 self.__initFromFile(other) 

78 else: 

79 # if the policy specified by other could not be loaded raise a runtime error. 

80 raise RuntimeError("A Policy could not be loaded from other:%s" % other) 

81 

82 def ppprint(self): 

83 """helper function for debugging, prints a policy out in a readable way in the debugger. 

84 

85 use: pdb> print myPolicyObject.pprint() 

86 :return: a prettyprint formatted string representing the policy 

87 """ 

88 import pprint 

89 return pprint.pformat(self.data, indent=2, width=1) 

90 

91 def __repr__(self): 

92 return self.data.__repr__() 

93 

94 def __initFromFile(self, path): 

95 """Load a file from path. If path is a list, will pick one to use, according to order specified 

96 by extensionPreference. 

97 

98 :param path: string or list of strings, to a persisted policy file. 

99 :param extensionPreference: the order in which to try to open files. Will use the first one that 

100 succeeds. 

101 :return: 

102 """ 

103 if path.endswith('yaml'): 

104 self.__initFromYamlFile(path) 

105 else: 

106 raise RuntimeError("Unhandled policy file type:%s" % path) 

107 

108 def __initFromYamlFile(self, path): 

109 """Opens a file at a given path and attempts to load it in from yaml. 

110 

111 :param path: 

112 :return: 

113 """ 

114 with open(path, 'r') as f: 

115 self.__initFromYaml(f) 

116 

117 def __initFromYaml(self, stream): 

118 """Loads a YAML policy from any readable stream that contains one. 

119 

120 :param stream: 

121 :return: 

122 """ 

123 # will raise yaml.YAMLError if there is an error loading the file. 

124 try: 

125 # PyYAML >=5.1 prefers a different loader 

126 loader = yaml.UnsafeLoader 

127 except AttributeError: 

128 loader = yaml.Loader 

129 self.data = yaml.load(stream, Loader=loader) 

130 return self 

131 

132 def __getitem__(self, name): 

133 data = self.data 

134 for key in name.split('.'): 

135 if data is None: 

136 return None 

137 if key in data: 

138 data = data[key] 

139 else: 

140 return None 

141 if isinstance(data, collections.abc.Mapping): 

142 data = Policy(data) 

143 return data 

144 

145 def __setitem__(self, name, value): 

146 if isinstance(value, collections.abc.Mapping): 

147 keys = name.split('.') 

148 d = {} 

149 cur = d 

150 for key in keys[0:-1]: 

151 cur[key] = {} 

152 cur = cur[key] 

153 cur[keys[-1]] = value 

154 self.update(d) 

155 data = self.data 

156 keys = name.split('.') 

157 for key in keys[0:-1]: 

158 data = data.setdefault(key, {}) 

159 data[keys[-1]] = value 

160 

161 def __contains__(self, key): 

162 d = self.data 

163 keys = key.split('.') 

164 for k in keys[0:-1]: 

165 if k in d: 

166 d = d[k] 

167 else: 

168 return False 

169 return keys[-1] in d 

170 

171 @staticmethod 

172 def defaultPolicyFile(productName, fileName, relativePath=None): 

173 """Get the path to a default policy file. 

174 

175 Determines a directory for the product specified by productName. Then Concatenates 

176 productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path 

177 to the default Policy file 

178 

179 @param productName (string) The name of the product that the default policy is installed as part of 

180 @param fileName (string) The name of the policy file. Can also include a path to the file relative to 

181 the directory where the product is installed. 

182 @param relativePath (string) The relative path from the directior where the product is installed to 

183 the location where the file (or the path to the file) is found. If None 

184 (default), the fileName argument is relative to the installation 

185 directory. 

186 """ 

187 basePath = lsst.utils.getPackageDir(productName) 

188 if not basePath: 

189 raise RuntimeError("No product installed for productName: %s" % basePath) 

190 if relativePath is not None: 

191 basePath = os.path.join(basePath, relativePath) 

192 fullFilePath = os.path.join(basePath, fileName) 

193 return fullFilePath 

194 

195 def update(self, other): 

196 """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested 

197 dict entirely. 

198 

199 For example, for the given code: 

200 foo = {'a': {'b': 1}} 

201 foo.update({'a': {'c': 2}}) 

202 

203 If foo is a dict, then after the update foo == {'a': {'c': 2}} 

204 But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}} 

205 """ 

206 def doUpdate(d, u): 

207 for k, v in u.items(): 

208 if isinstance(d, collections.abc.Mapping): 

209 if isinstance(v, collections.abc.Mapping): 

210 r = doUpdate(d.get(k, {}), v) 

211 d[k] = r 

212 else: 

213 d[k] = u[k] 

214 else: 

215 d = {k: u[k]} 

216 return d 

217 doUpdate(self.data, other) 

218 

219 def merge(self, other): 

220 """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and 

221 values that already exist in self will NOT be overwritten. 

222 

223 :param other: 

224 :return: 

225 """ 

226 otherCopy = copy.deepcopy(other) 

227 otherCopy.update(self) 

228 self.data = otherCopy.data 

229 

230 def names(self, topLevelOnly=False): 

231 """Get the dot-delimited name of all the keys in the hierarchy. 

232 NOTE: this is different than the built-in method dict.keys, which will return only the first level 

233 keys. 

234 """ 

235 if topLevelOnly: 

236 return list(self.keys()) 

237 

238 def getKeys(d, keys, base): 

239 for key in d: 

240 val = d[key] 

241 levelKey = base + '.' + key if base is not None else key 

242 keys.append(levelKey) 

243 if isinstance(val, collections.abc.Mapping): 

244 getKeys(val, keys, levelKey) 

245 keys = [] 

246 getKeys(self.data, keys, None) 

247 return keys 

248 

249 def asArray(self, name): 

250 """Get a value as an array. May contain one or more elements. 

251 

252 :param key: 

253 :return: 

254 """ 

255 val = self.get(name) 

256 if isinstance(val, str): 

257 val = [val] 

258 elif not isinstance(val, collections.abc.Container): 

259 val = [val] 

260 return val 

261 

262 # Deprecated methods that mimic pex_policy api. 

263 # These are supported (for now), but callers should use the dict api. 

264 

265 def getValue(self, name): 

266 """Get the value for a parameter name/key. See class notes about dot-delimited access. 

267 

268 :param name: 

269 :return: the value for the given name. 

270 """ 

271 warnings.warn_explicit("Deprecated. Use []", DeprecationWarning) 

272 return self[name] 

273 

274 def setValue(self, name, value): 

275 """Set the value for a parameter name/key. See class notes about dot-delimited access. 

276 

277 :param name: 

278 :return: None 

279 """ 

280 warnings.warn("Deprecated. Use []", DeprecationWarning) 

281 self[name] = value 

282 

283 def mergeDefaults(self, other): 

284 """For any keys in other that are not present in self, sets that key and its value into self. 

285 

286 :param other: another Policy 

287 :return: None 

288 """ 

289 warnings.warn("Deprecated. Use .merge()", DeprecationWarning) 

290 self.merge(other) 

291 

292 def exists(self, key): 

293 """Query if a key exists in this Policy 

294 

295 :param key: 

296 :return: True if the key exists, else false. 

297 """ 

298 warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning) 

299 return key in self 

300 

301 def getString(self, key): 

302 """Get the string value of a key. 

303 

304 :param key: 

305 :return: the value for key 

306 """ 

307 warnings.warn("Deprecated. Use []", DeprecationWarning) 

308 return str(self[key]) 

309 

310 def getBool(self, key): 

311 """Get the value of a key. 

312 

313 :param key: 

314 :return: the value for key 

315 """ 

316 warnings.warn("Deprecated. Use []", DeprecationWarning) 

317 return bool(self[key]) 

318 

319 def getPolicy(self, key): 

320 """Get a subpolicy. 

321 

322 :param key: 

323 :return: 

324 """ 

325 warnings.warn("Deprecated. Use []", DeprecationWarning) 

326 return self[key] 

327 

328 def getStringArray(self, key): 

329 """Get a value as an array. May contain one or more elements. 

330 

331 :param key: 

332 :return: 

333 """ 

334 warnings.warn("Deprecated. Use asArray()", DeprecationWarning) 

335 val = self.get(key) 

336 if isinstance(val, str): 

337 val = [val] 

338 elif not isinstance(val, collections.abc.Container): 

339 val = [val] 

340 return val 

341 

342 def __lt__(self, other): 

343 if isinstance(other, Policy): 

344 other = other.data 

345 return self.data < other 

346 

347 def __le__(self, other): 

348 if isinstance(other, Policy): 

349 other = other.data 

350 return self.data <= other 

351 

352 def __eq__(self, other): 

353 if isinstance(other, Policy): 

354 other = other.data 

355 return self.data == other 

356 

357 def __ne__(self, other): 

358 if isinstance(other, Policy): 

359 other = other.data 

360 return self.data != other 

361 

362 def __gt__(self, other): 

363 if isinstance(other, Policy): 

364 other = other.data 

365 return self.data > other 

366 

367 def __ge__(self, other): 

368 if isinstance(other, Policy): 

369 other = other.data 

370 return self.data >= other 

371 

372 ####### 

373 # i/o # 

374 

375 def dump(self, output): 

376 """Writes the policy to a yaml stream. 

377 

378 :param stream: 

379 :return: 

380 """ 

381 # First a set of known keys is handled and written to the stream in a specific order for readability. 

382 # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to 

383 # the stream. 

384 data = copy.copy(self.data) 

385 keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera', 

386 'exposures', 'calibrations', 'datasets'] 

387 for key in keys: 

388 try: 

389 yaml.safe_dump({key: data.pop(key)}, output, default_flow_style=False) 

390 output.write('\n') 

391 except KeyError: 

392 pass 

393 if data: 

394 yaml.safe_dump(data, output, default_flow_style=False) 

395 

396 def dumpToFile(self, path): 

397 """Writes the policy to a file. 

398 

399 :param path: 

400 :return: 

401 """ 

402 with open(path, 'w') as f: 

403 self.dump(f)