Coverage for python/lsst/daf/persistence/policy.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
3#
4# LSST Data Management System
5# Copyright 2015 LSST Corporation.
6#
7# This product includes software developed by the
8# LSST Project (http://www.lsst.org/).
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the LSST License Statement and
21# the GNU General Public License along with this program. If not,
22# see <http://www.lsstcorp.org/LegalNotices/>.
23#
24import collections
25import collections.abc
26import copy
27import os
28import warnings
29import yaml
31import lsst.pex.policy as pexPolicy
32import lsst.utils
34from yaml.representer import Representer
35yaml.add_representer(collections.defaultdict, Representer.represent_dict)
38# UserDict and yaml have defined metaclasses and Python 3 does not allow multiple
39# inheritance of classes with distinct metaclasses. We therefore have to
40# create a new baseclass that Policy can inherit from.
41class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)):
42 pass
45class _PolicyBase(collections.UserDict, yaml.YAMLObject, metaclass=_PolicyMeta):
46 pass
49class Policy(_PolicyBase):
50 """Policy implements a datatype that is used by Butler for configuration parameters.
51 It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be
52 initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next:
53 Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation.
54 That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either
55 of these syntaxes may be used.
57 Storage formats supported:
58 - yaml: read and write is supported.
59 - pex policy: read is supported, although this is deprecated and will at some point be removed.
60 """
62 def __init__(self, other=None):
63 """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways:
64 other (string) Treated as a path to a policy file on disk. Must end with '.paf' or '.yaml'.
65 other (Pex Policy) Initializes this Policy with the values in the passed-in Pex Policy.
66 other (Policy) Copies the other Policy's values into this one.
67 other (dict) Copies the values from the dict into this Policy.
68 """
69 collections.UserDict.__init__(self)
71 if other is None:
72 return
74 if isinstance(other, collections.abc.Mapping):
75 self.update(other)
76 elif isinstance(other, Policy):
77 self.data = copy.deepcopy(other.data)
78 elif isinstance(other, str):
79 # if other is a string, assume it is a file path.
80 self.__initFromFile(other)
81 elif isinstance(other, pexPolicy.Policy):
82 # if other is an instance of a Pex Policy, load it accordingly.
83 self.__initFromPexPolicy(other)
84 else:
85 # if the policy specified by other could not be loaded raise a runtime error.
86 raise RuntimeError("A Policy could not be loaded from other:%s" % other)
88 def ppprint(self):
89 """helper function for debugging, prints a policy out in a readable way in the debugger.
91 use: pdb> print myPolicyObject.pprint()
92 :return: a prettyprint formatted string representing the policy
93 """
94 import pprint
95 return pprint.pformat(self.data, indent=2, width=1)
97 def __repr__(self):
98 return self.data.__repr__()
100 def __initFromFile(self, path):
101 """Load a file from path. If path is a list, will pick one to use, according to order specified
102 by extensionPreference.
104 :param path: string or list of strings, to a persisted policy file.
105 :param extensionPreference: the order in which to try to open files. Will use the first one that
106 succeeds.
107 :return:
108 """
109 policy = None
110 if path.endswith('yaml'):
111 self.__initFromYamlFile(path)
112 elif path.endswith('paf'):
113 policy = pexPolicy.Policy.createPolicy(path)
114 self.__initFromPexPolicy(policy)
115 else:
116 raise RuntimeError("Unhandled policy file type:%s" % path)
118 def __initFromPexPolicy(self, pexPolicy):
119 """Load values from a pex policy.
121 :param pexPolicy:
122 :return:
123 """
124 names = pexPolicy.names()
125 names.sort()
126 for name in names:
127 if pexPolicy.getValueType(name) == pexPolicy.POLICY:
128 if name in self:
129 continue
130 else:
131 self[name] = {}
132 else:
133 if pexPolicy.isArray(name):
134 self[name] = pexPolicy.getArray(name)
135 else:
136 self[name] = pexPolicy.get(name)
137 return self
139 def __initFromYamlFile(self, path):
140 """Opens a file at a given path and attempts to load it in from yaml.
142 :param path:
143 :return:
144 """
145 with open(path, 'r') as f:
146 self.__initFromYaml(f)
148 def __initFromYaml(self, stream):
149 """Loads a YAML policy from any readable stream that contains one.
151 :param stream:
152 :return:
153 """
154 # will raise yaml.YAMLError if there is an error loading the file.
155 try:
156 # PyYAML >=5.1 prefers a different loader
157 loader = yaml.FullLoader
158 except AttributeError:
159 loader = yaml.Loader
160 self.data = yaml.load(stream, Loader=loader)
161 return self
163 def __getitem__(self, name):
164 data = self.data
165 for key in name.split('.'):
166 if data is None:
167 return None
168 if key in data:
169 data = data[key]
170 else:
171 return None
172 if isinstance(data, collections.abc.Mapping):
173 data = Policy(data)
174 return data
176 def __setitem__(self, name, value):
177 if isinstance(value, collections.abc.Mapping):
178 keys = name.split('.')
179 d = {}
180 cur = d
181 for key in keys[0:-1]:
182 cur[key] = {}
183 cur = cur[key]
184 cur[keys[-1]] = value
185 self.update(d)
186 data = self.data
187 keys = name.split('.')
188 for key in keys[0:-1]:
189 data = data.setdefault(key, {})
190 data[keys[-1]] = value
192 def __contains__(self, key):
193 d = self.data
194 keys = key.split('.')
195 for k in keys[0:-1]:
196 if k in d:
197 d = d[k]
198 else:
199 return False
200 return keys[-1] in d
202 @staticmethod
203 def defaultPolicyFile(productName, fileName, relativePath=None):
204 """Get the path to a default policy file.
206 Determines a directory for the product specified by productName. Then Concatenates
207 productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path
208 to the default Policy file
210 @param productName (string) The name of the product that the default policy is installed as part of
211 @param fileName (string) The name of the policy file. Can also include a path to the file relative to
212 the directory where the product is installed.
213 @param relativePath (string) The relative path from the directior where the product is installed to
214 the location where the file (or the path to the file) is found. If None
215 (default), the fileName argument is relative to the installation
216 directory.
217 """
218 basePath = lsst.utils.getPackageDir(productName)
219 if not basePath:
220 raise RuntimeError("No product installed for productName: %s" % basePath)
221 if relativePath is not None:
222 basePath = os.path.join(basePath, relativePath)
223 fullFilePath = os.path.join(basePath, fileName)
224 return fullFilePath
226 def update(self, other):
227 """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested
228 dict entirely.
230 For example, for the given code:
231 foo = {'a': {'b': 1}}
232 foo.update({'a': {'c': 2}})
234 If foo is a dict, then after the update foo == {'a': {'c': 2}}
235 But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}}
236 """
237 def doUpdate(d, u):
238 for k, v in u.items():
239 if isinstance(d, collections.abc.Mapping):
240 if isinstance(v, collections.abc.Mapping):
241 r = doUpdate(d.get(k, {}), v)
242 d[k] = r
243 else:
244 d[k] = u[k]
245 else:
246 d = {k: u[k]}
247 return d
248 doUpdate(self.data, other)
250 def merge(self, other):
251 """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and
252 values that already exist in self will NOT be overwritten.
254 :param other:
255 :return:
256 """
257 otherCopy = copy.deepcopy(other)
258 otherCopy.update(self)
259 self.data = otherCopy.data
261 def names(self, topLevelOnly=False):
262 """Get the dot-delimited name of all the keys in the hierarchy.
263 NOTE: this is different than the built-in method dict.keys, which will return only the first level
264 keys.
265 """
266 if topLevelOnly:
267 return list(self.keys())
269 def getKeys(d, keys, base):
270 for key in d:
271 val = d[key]
272 levelKey = base + '.' + key if base is not None else key
273 keys.append(levelKey)
274 if isinstance(val, collections.abc.Mapping):
275 getKeys(val, keys, levelKey)
276 keys = []
277 getKeys(self.data, keys, None)
278 return keys
280 def asArray(self, name):
281 """Get a value as an array. May contain one or more elements.
283 :param key:
284 :return:
285 """
286 val = self.get(name)
287 if isinstance(val, str):
288 val = [val]
289 elif not isinstance(val, collections.abc.Container):
290 val = [val]
291 return val
293 # Deprecated methods that mimic pex_policy api.
294 # These are supported (for now), but callers should use the dict api.
296 def getValue(self, name):
297 """Get the value for a parameter name/key. See class notes about dot-delimited access.
299 :param name:
300 :return: the value for the given name.
301 """
302 warnings.warn_explicit("Deprecated. Use []", DeprecationWarning)
303 return self[name]
305 def setValue(self, name, value):
306 """Set the value for a parameter name/key. See class notes about dot-delimited access.
308 :param name:
309 :return: None
310 """
311 warnings.warn("Deprecated. Use []", DeprecationWarning)
312 self[name] = value
314 def mergeDefaults(self, other):
315 """For any keys in other that are not present in self, sets that key and its value into self.
317 :param other: another Policy
318 :return: None
319 """
320 warnings.warn("Deprecated. Use .merge()", DeprecationWarning)
321 self.merge(other)
323 def exists(self, key):
324 """Query if a key exists in this Policy
326 :param key:
327 :return: True if the key exists, else false.
328 """
329 warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning)
330 return key in self
332 def getString(self, key):
333 """Get the string value of a key.
335 :param key:
336 :return: the value for key
337 """
338 warnings.warn("Deprecated. Use []", DeprecationWarning)
339 return str(self[key])
341 def getBool(self, key):
342 """Get the value of a key.
344 :param key:
345 :return: the value for key
346 """
347 warnings.warn("Deprecated. Use []", DeprecationWarning)
348 return bool(self[key])
350 def getPolicy(self, key):
351 """Get a subpolicy.
353 :param key:
354 :return:
355 """
356 warnings.warn("Deprecated. Use []", DeprecationWarning)
357 return self[key]
359 def getStringArray(self, key):
360 """Get a value as an array. May contain one or more elements.
362 :param key:
363 :return:
364 """
365 warnings.warn("Deprecated. Use asArray()", DeprecationWarning)
366 val = self.get(key)
367 if isinstance(val, str):
368 val = [val]
369 elif not isinstance(val, collections.abc.Container):
370 val = [val]
371 return val
373 def __lt__(self, other):
374 if isinstance(other, Policy):
375 other = other.data
376 return self.data < other
378 def __le__(self, other):
379 if isinstance(other, Policy):
380 other = other.data
381 return self.data <= other
383 def __eq__(self, other):
384 if isinstance(other, Policy):
385 other = other.data
386 return self.data == other
388 def __ne__(self, other):
389 if isinstance(other, Policy):
390 other = other.data
391 return self.data != other
393 def __gt__(self, other):
394 if isinstance(other, Policy):
395 other = other.data
396 return self.data > other
398 def __ge__(self, other):
399 if isinstance(other, Policy):
400 other = other.data
401 return self.data >= other
403 #######
404 # i/o #
406 def dump(self, output):
407 """Writes the policy to a yaml stream.
409 :param stream:
410 :return:
411 """
412 # First a set of known keys is handled and written to the stream in a specific order for readability.
413 # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to
414 # the stream.
415 data = copy.copy(self.data)
416 keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera',
417 'exposures', 'calibrations', 'datasets']
418 for key in keys:
419 try:
420 yaml.safe_dump({key: data.pop(key)}, output, default_flow_style=False)
421 output.write('\n')
422 except KeyError:
423 pass
424 if data:
425 yaml.safe_dump(data, output, default_flow_style=False)
427 def dumpToFile(self, path):
428 """Writes the policy to a file.
430 :param path:
431 :return:
432 """
433 with open(path, 'w') as f:
434 self.dump(f)