lsst.daf.persistence  16.0-12-g4477fe7+1
policy.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2015 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from builtins import str
25 from past.builtins import basestring
26 from future.utils import with_metaclass
27 
28 import collections
29 import copy
30 import os
31 import sys
32 import warnings
33 import yaml
34 
35 import lsst.pex.policy as pexPolicy
36 import lsst.utils
37 
38 from yaml.representer import Representer
39 yaml.add_representer(collections.defaultdict, Representer.represent_dict)
40 
41 # UserDict and yaml have defined metaclasses and Python 3 does not allow multiple
42 # inheritance of classes with distinct metaclasses. We therefore have to
43 # create a new baseclass that Policy can inherit from. This is because the metaclass
44 # syntax differs between versions
45 
46 if sys.version_info[0] >= 3:
47  class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)):
48  pass
49 
50  class _PolicyBase(with_metaclass(_PolicyMeta, collections.UserDict, yaml.YAMLObject)):
51  pass
52 else:
53  class _PolicyBase(collections.UserDict, yaml.YAMLObject):
54  pass
55 
56 
58  """Policy implements a datatype that is used by Butler for configuration parameters.
59  It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be
60  initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next:
61  Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation.
62  That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either
63  of these syntaxes may be used.
64 
65  Storage formats supported:
66  - yaml: read and write is supported.
67  - pex policy: read is supported, although this is deprecated and will at some point be removed.
68  """
69 
70  def __init__(self, other=None):
71  """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways:
72  other (string) Treated as a path to a policy file on disk. Must end with '.paf' or '.yaml'.
73  other (Pex Policy) Initializes this Policy with the values in the passed-in Pex Policy.
74  other (Policy) Copies the other Policy's values into this one.
75  other (dict) Copies the values from the dict into this Policy.
76  """
77  collections.UserDict.__init__(self)
78 
79  if other is None:
80  return
81 
82  if isinstance(other, collections.Mapping):
83  self.update(other)
84  elif isinstance(other, Policy):
85  self.data = copy.deepcopy(other.data)
86  elif isinstance(other, basestring):
87  # if other is a string, assume it is a file path.
88  self.__initFromFile(other)
89  elif isinstance(other, pexPolicy.Policy):
90  # if other is an instance of a Pex Policy, load it accordingly.
91  self.__initFromPexPolicy(other)
92  else:
93  # if the policy specified by other could not be loaded raise a runtime error.
94  raise RuntimeError("A Policy could not be loaded from other:%s" % other)
95 
96  def ppprint(self):
97  """helper function for debugging, prints a policy out in a readable way in the debugger.
98 
99  use: pdb> print myPolicyObject.pprint()
100  :return: a prettyprint formatted string representing the policy
101  """
102  import pprint
103  return pprint.pformat(self.data, indent=2, width=1)
104 
105  def __repr__(self):
106  return self.data.__repr__()
107 
108  def __initFromFile(self, path):
109  """Load a file from path. If path is a list, will pick one to use, according to order specified
110  by extensionPreference.
111 
112  :param path: string or list of strings, to a persisted policy file.
113  :param extensionPreference: the order in which to try to open files. Will use the first one that
114  succeeds.
115  :return:
116  """
117  policy = None
118  if path.endswith('yaml'):
119  self.__initFromYamlFile(path)
120  elif path.endswith('paf'):
121  policy = pexPolicy.Policy.createPolicy(path)
122  self.__initFromPexPolicy(policy)
123  else:
124  raise RuntimeError("Unhandled policy file type:%s" % path)
125 
126  def __initFromPexPolicy(self, pexPolicy):
127  """Load values from a pex policy.
128 
129  :param pexPolicy:
130  :return:
131  """
132  names = pexPolicy.names()
133  names.sort()
134  for name in names:
135  if pexPolicy.getValueType(name) == pexPolicy.POLICY:
136  if name in self:
137  continue
138  else:
139  self[name] = {}
140  else:
141  if pexPolicy.isArray(name):
142  self[name] = pexPolicy.getArray(name)
143  else:
144  self[name] = pexPolicy.get(name)
145  return self
146 
147  def __initFromYamlFile(self, path):
148  """Opens a file at a given path and attempts to load it in from yaml.
149 
150  :param path:
151  :return:
152  """
153  with open(path, 'r') as f:
154  self.__initFromYaml(f)
155 
156  def __initFromYaml(self, stream):
157  """Loads a YAML policy from any readable stream that contains one.
158 
159  :param stream:
160  :return:
161  """
162  # will raise yaml.YAMLError if there is an error loading the file.
163  self.data = yaml.load(stream)
164  return self
165 
166  def __getitem__(self, name):
167  data = self.data
168  for key in name.split('.'):
169  if data is None:
170  return None
171  if key in data:
172  data = data[key]
173  else:
174  return None
175  if isinstance(data, collections.Mapping):
176  data = Policy(data)
177  return data
178 
179  def __setitem__(self, name, value):
180  if isinstance(value, collections.Mapping):
181  keys = name.split('.')
182  d = {}
183  cur = d
184  for key in keys[0:-1]:
185  cur[key] = {}
186  cur = cur[key]
187  cur[keys[-1]] = value
188  self.update(d)
189  data = self.data
190  keys = name.split('.')
191  for key in keys[0:-1]:
192  data = data.setdefault(key, {})
193  data[keys[-1]] = value
194 
195  def __contains__(self, key):
196  d = self.data
197  keys = key.split('.')
198  for k in keys[0:-1]:
199  if k in d:
200  d = d[k]
201  else:
202  return False
203  return keys[-1] in d
204 
205  @staticmethod
206  def defaultPolicyFile(productName, fileName, relativePath=None):
207  """Get the path to a default policy file.
208 
209  Determines a directory for the product specified by productName. Then Concatenates
210  productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path
211  to the default Policy file
212 
213  @param productName (string) The name of the product that the default policy is installed as part of
214  @param fileName (string) The name of the policy file. Can also include a path to the file relative to
215  the directory where the product is installed.
216  @param relativePath (string) The relative path from the directior where the product is installed to
217  the location where the file (or the path to the file) is found. If None
218  (default), the fileName argument is relative to the installation
219  directory.
220  """
221  basePath = lsst.utils.getPackageDir(productName)
222  if not basePath:
223  raise RuntimeError("No product installed for productName: %s" % basePath)
224  if relativePath is not None:
225  basePath = os.path.join(basePath, relativePath)
226  fullFilePath = os.path.join(basePath, fileName)
227  return fullFilePath
228 
229  def update(self, other):
230  """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested
231  dict entirely.
232 
233  For example, for the given code:
234  foo = {'a': {'b': 1}}
235  foo.update({'a': {'c': 2}})
236 
237  If foo is a dict, then after the update foo == {'a': {'c': 2}}
238  But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}}
239  """
240  def doUpdate(d, u):
241  for k, v in u.items():
242  if isinstance(d, collections.Mapping):
243  if isinstance(v, collections.Mapping):
244  r = doUpdate(d.get(k, {}), v)
245  d[k] = r
246  else:
247  d[k] = u[k]
248  else:
249  d = {k: u[k]}
250  return d
251  doUpdate(self.data, other)
252 
253  def merge(self, other):
254  """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and
255  values that already exist in self will NOT be overwritten.
256 
257  :param other:
258  :return:
259  """
260  otherCopy = copy.deepcopy(other)
261  otherCopy.update(self)
262  self.data = otherCopy.data
263 
264  def names(self, topLevelOnly=False):
265  """Get the dot-delimited name of all the keys in the hierarchy.
266  NOTE: this is different than the built-in method dict.keys, which will return only the first level
267  keys.
268  """
269  if topLevelOnly:
270  return list(self.keys())
271 
272  def getKeys(d, keys, base):
273  for key in d:
274  val = d[key]
275  levelKey = base + '.' + key if base is not None else key
276  keys.append(levelKey)
277  if isinstance(val, collections.Mapping):
278  getKeys(val, keys, levelKey)
279  keys = []
280  getKeys(self.data, keys, None)
281  return keys
282 
283  def asArray(self, name):
284  """Get a value as an array. May contain one or more elements.
285 
286  :param key:
287  :return:
288  """
289  val = self.get(name)
290  if isinstance(val, basestring):
291  val = [val]
292  elif not isinstance(val, collections.Container):
293  val = [val]
294  return val
295 
296  # Deprecated methods that mimic pex_policy api.
297  # These are supported (for now), but callers should use the dict api.
298 
299  def getValue(self, name):
300  """Get the value for a parameter name/key. See class notes about dot-delimited access.
301 
302  :param name:
303  :return: the value for the given name.
304  """
305  warnings.warn_explicit("Deprecated. Use []", DeprecationWarning)
306  return self[name]
307 
308  def setValue(self, name, value):
309  """Set the value for a parameter name/key. See class notes about dot-delimited access.
310 
311  :param name:
312  :return: None
313  """
314  warnings.warn("Deprecated. Use []", DeprecationWarning)
315  self[name] = value
316 
317  def mergeDefaults(self, other):
318  """For any keys in other that are not present in self, sets that key and its value into self.
319 
320  :param other: another Policy
321  :return: None
322  """
323  warnings.warn("Deprecated. Use .merge()", DeprecationWarning)
324  self.merge(other)
325 
326  def exists(self, key):
327  """Query if a key exists in this Policy
328 
329  :param key:
330  :return: True if the key exists, else false.
331  """
332  warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning)
333  return key in self
334 
335  def getString(self, key):
336  """Get the string value of a key.
337 
338  :param key:
339  :return: the value for key
340  """
341  warnings.warn("Deprecated. Use []", DeprecationWarning)
342  return str(self[key])
343 
344  def getBool(self, key):
345  """Get the value of a key.
346 
347  :param key:
348  :return: the value for key
349  """
350  warnings.warn("Deprecated. Use []", DeprecationWarning)
351  return bool(self[key])
352 
353  def getPolicy(self, key):
354  """Get a subpolicy.
355 
356  :param key:
357  :return:
358  """
359  warnings.warn("Deprecated. Use []", DeprecationWarning)
360  return self[key]
361 
362  def getStringArray(self, key):
363  """Get a value as an array. May contain one or more elements.
364 
365  :param key:
366  :return:
367  """
368  warnings.warn("Deprecated. Use asArray()", DeprecationWarning)
369  val = self.get(key)
370  if isinstance(val, basestring):
371  val = [val]
372  elif not isinstance(val, collections.Container):
373  val = [val]
374  return val
375 
376  def __lt__(self, other):
377  if isinstance(other, Policy):
378  other = other.data
379  return self.data < other
380 
381  def __le__(self, other):
382  if isinstance(other, Policy):
383  other = other.data
384  return self.data <= other
385 
386  def __eq__(self, other):
387  if isinstance(other, Policy):
388  other = other.data
389  return self.data == other
390 
391  def __ne__(self, other):
392  if isinstance(other, Policy):
393  other = other.data
394  return self.data != other
395 
396  def __gt__(self, other):
397  if isinstance(other, Policy):
398  other = other.data
399  return self.data > other
400 
401  def __ge__(self, other):
402  if isinstance(other, Policy):
403  other = other.data
404  return self.data >= other
405 
406 
408 
409  def dump(self, output):
410  """Writes the policy to a yaml stream.
411 
412  :param stream:
413  :return:
414  """
415  # First a set of known keys is handled and written to the stream in a specific order for readability.
416  # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to
417  # the stream.
418  data = copy.copy(self.data)
419  keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera',
420  'exposures', 'calibrations', 'datasets']
421  for key in keys:
422  try:
423  yaml.safe_dump({key: data.pop(key)}, output, default_flow_style=False)
424  output.write('\n')
425  except KeyError:
426  pass
427  if data:
428  yaml.safe_dump(data, output, default_flow_style=False)
429 
430  def dumpToFile(self, path):
431  """Writes the policy to a file.
432 
433  :param path:
434  :return:
435  """
436  with open(path, 'w') as f:
437  self.dump(f)
def __initFromYaml(self, stream)
Definition: policy.py:156
def __initFromYamlFile(self, path)
Definition: policy.py:147
def dump(self, output)
i/o #
Definition: policy.py:409
def __initFromPexPolicy(self, pexPolicy)
Definition: policy.py:126
def __setitem__(self, name, value)
Definition: policy.py:179
std::string getPackageDir(std::string const &packageName)
def __init__(self, other=None)
Definition: policy.py:70
def defaultPolicyFile(productName, fileName, relativePath=None)
Definition: policy.py:206
def mergeDefaults(self, other)
Definition: policy.py:317
def names(self, topLevelOnly=False)
Definition: policy.py:264
def setValue(self, name, value)
Definition: policy.py:308