lsst.daf.persistence  13.0-17-gd5d205a+2
 All Classes Namespaces Files Functions Variables Typedefs Friends Macros
policy.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 
3 #
4 # LSST Data Management System
5 # Copyright 2015 LSST Corporation.
6 #
7 # This product includes software developed by the
8 # LSST Project (http://www.lsst.org/).
9 #
10 # This program is free software: you can redistribute it and/or modify
11 # it under the terms of the GNU General Public License as published by
12 # the Free Software Foundation, either version 3 of the License, or
13 # (at your option) any later version.
14 #
15 # This program is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 # GNU General Public License for more details.
19 #
20 # You should have received a copy of the LSST License Statement and
21 # the GNU General Public License along with this program. If not,
22 # see <http://www.lsstcorp.org/LegalNotices/>.
23 #
24 from builtins import str
25 from past.builtins import basestring
26 from future.utils import with_metaclass
27 from future.standard_library import install_aliases
28 install_aliases()
29 
30 import collections
31 import copy
32 import os
33 import sys
34 import warnings
35 import yaml
36 
37 from yaml.representer import Representer
38 yaml.add_representer(collections.defaultdict, Representer.represent_dict)
39 
40 import lsst.pex.policy as pexPolicy
41 import lsst.utils
42 
43 # UserDict and yaml have defined metaclasses and Python 3 does not allow multiple
44 # inheritance of classes with distinct metaclasses. We therefore have to
45 # create a new baseclass that Policy can inherit from. This is because the metaclass
46 # syntax differs between versions
47 
48 if sys.version_info[0] >= 3:
49  class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)):
50  pass
51 
52  class _PolicyBase(with_metaclass(_PolicyMeta, collections.UserDict, yaml.YAMLObject)):
53  pass
54 else:
55  class _PolicyBase(collections.UserDict, yaml.YAMLObject):
56  pass
57 
58 
60  """Policy implements a datatype that is used by Butler for configuration parameters.
61  It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be
62  initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next:
63  Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation.
64  That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either
65  of these syntaxes may be used.
66 
67  Storage formats supported:
68  - yaml: read and write is supported.
69  - pex policy: read is supported, although this is deprecated and will at some point be removed.
70  """
71 
72  def __init__(self, other=None):
73  """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways:
74  other (string) Treated as a path to a policy file on disk. Must end with '.paf' or '.yaml'.
75  other (Pex Policy) Initializes this Policy with the values in the passed-in Pex Policy.
76  other (Policy) Copies the other Policy's values into this one.
77  other (dict) Copies the values from the dict into this Policy.
78  """
79  collections.UserDict.__init__(self)
80 
81  if other is None:
82  return
83 
84  if isinstance(other, collections.Mapping):
85  self.update(other)
86  elif isinstance(other, Policy):
87  self.data = copy.deepcopy(other.data)
88  elif isinstance(other, basestring):
89  # if other is a string, assume it is a file path.
90  self.__initFromFile(other)
91  elif isinstance(other, pexPolicy.Policy):
92  # if other is an instance of a Pex Policy, load it accordingly.
93  self.__initFromPexPolicy(other)
94  else:
95  # if the policy specified by other could not be loaded raise a runtime error.
96  raise RuntimeError("A Policy could not be loaded from other:%s" % other)
97 
98  def ppprint(self):
99  """helper function for debugging, prints a policy out in a readable way in the debugger.
100 
101  use: pdb> print myPolicyObject.pprint()
102  :return: a prettyprint formatted string representing the policy
103  """
104  import pprint
105  return pprint.pformat(self.data, indent=2, width=1)
106 
107  def __repr__(self):
108  return self.data.__repr__()
109 
110  def __initFromFile(self, path):
111  """Load a file from path. If path is a list, will pick one to use, according to order specified
112  by extensionPreference.
113 
114  :param path: string or list of strings, to a persisted policy file.
115  :param extensionPreference: the order in which to try to open files. Will use the first one that
116  succeeds.
117  :return:
118  """
119  policy = None
120  if path.endswith('yaml'):
121  self.__initFromYamlFile(path)
122  elif path.endswith('paf'):
123  policy = pexPolicy.Policy.createPolicy(path)
124  self.__initFromPexPolicy(policy)
125  else:
126  raise RuntimeError("Unhandled policy file type:%s" % path)
127 
128  def __initFromPexPolicy(self, pexPolicy):
129  """Load values from a pex policy.
130 
131  :param pexPolicy:
132  :return:
133  """
134  names = pexPolicy.names()
135  names.sort()
136  for name in names:
137  if pexPolicy.getValueType(name) == pexPolicy.POLICY:
138  if name in self:
139  continue
140  else:
141  self[name] = {}
142  else:
143  if pexPolicy.isArray(name):
144  self[name] = pexPolicy.getArray(name)
145  else:
146  self[name] = pexPolicy.get(name)
147  return self
148 
149  def __initFromYamlFile(self, path):
150  """Opens a file at a given path and attempts to load it in from yaml.
151 
152  :param path:
153  :return:
154  """
155  with open(path, 'r') as f:
156  self.__initFromYaml(f)
157 
158  def __initFromYaml(self, stream):
159  """Loads a YAML policy from any readable stream that contains one.
160 
161  :param stream:
162  :return:
163  """
164  # will raise yaml.YAMLError if there is an error loading the file.
165  self.data = yaml.load(stream)
166  return self
167 
168  def __getitem__(self, name):
169  data = self.data
170  for key in name.split('.'):
171  if data is None:
172  return None
173  if key in data:
174  data = data[key]
175  else:
176  return None
177  if isinstance(data, collections.Mapping):
178  data = Policy(data)
179  return data
180 
181  def __setitem__(self, name, value):
182  if isinstance(value, collections.Mapping):
183  keys = name.split('.')
184  d = {}
185  cur = d
186  for key in keys[0:-1]:
187  cur[key] = {}
188  cur = cur[key]
189  cur[keys[-1]] = value
190  self.update(d)
191  data = self.data
192  keys = name.split('.')
193  for key in keys[0:-1]:
194  data = data.setdefault(key, {})
195  data[keys[-1]] = value
196 
197  def __contains__(self, key):
198  d = self.data
199  keys = key.split('.')
200  for k in keys[0:-1]:
201  if k in d:
202  d = d[k]
203  else:
204  return False
205  return keys[-1] in d
206 
207  @staticmethod
208  def defaultPolicyFile(productName, fileName, relativePath=None):
209  """Get the path to a default policy file.
210 
211  Determines a directory for the product specified by productName. Then Concatenates
212  productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path
213  to the default Policy file
214 
215  @param productName (string) The name of the product that the default policy is installed as part of
216  @param fileName (string) The name of the policy file. Can also include a path to the file relative to
217  the directory where the product is installed.
218  @param relativePath (string) The relative path from the directior where the product is installed to
219  the location where the file (or the path to the file) is found. If None
220  (default), the fileName argument is relative to the installation
221  directory.
222  """
223  basePath = lsst.utils.getPackageDir(productName)
224  if not basePath:
225  raise RuntimeError("No product installed for productName: %s" % basePath)
226  if relativePath is not None:
227  basePath = os.path.join(basePath, relativePath)
228  fullFilePath = os.path.join(basePath, fileName)
229  return fullFilePath
230 
231  def update(self, other):
232  """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested
233  dict entirely.
234 
235  For example, for the given code:
236  foo = {'a': {'b': 1}}
237  foo.update({'a': {'c': 2}})
238 
239  If foo is a dict, then after the update foo == {'a': {'c': 2}}
240  But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}}
241  """
242  def doUpdate(d, u):
243  for k, v in u.items():
244  if isinstance(d, collections.Mapping):
245  if isinstance(v, collections.Mapping):
246  r = doUpdate(d.get(k, {}), v)
247  d[k] = r
248  else:
249  d[k] = u[k]
250  else:
251  d = {k: u[k]}
252  return d
253  doUpdate(self.data, other)
254 
255  def merge(self, other):
256  """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and
257  values that already exist in self will NOT be overwritten.
258 
259  :param other:
260  :return:
261  """
262  otherCopy = copy.deepcopy(other)
263  otherCopy.update(self)
264  self.data = otherCopy.data
265 
266  def names(self, topLevelOnly=False):
267  """Get the dot-delimited name of all the keys in the hierarchy.
268  NOTE: this is different than the built-in method dict.keys, which will return only the first level
269  keys.
270  """
271  if topLevelOnly:
272  return list(self.keys())
273 
274  def getKeys(d, keys, base):
275  for key in d:
276  val = d[key]
277  levelKey = base + '.' + key if base is not None else key
278  keys.append(levelKey)
279  if isinstance(val, collections.Mapping):
280  getKeys(val, keys, levelKey)
281  keys = []
282  getKeys(self.data, keys, None)
283  return keys
284 
285  def asArray(self, name):
286  """Get a value as an array. May contain one or more elements.
287 
288  :param key:
289  :return:
290  """
291  val = self.get(name)
292  if isinstance(val, basestring):
293  val = [val]
294  elif not isinstance(val, collections.Container):
295  val = [val]
296  return val
297 
298  # Deprecated methods that mimic pex_policy api.
299  # These are supported (for now), but callers should use the dict api.
300 
301  def getValue(self, name):
302  """Get the value for a parameter name/key. See class notes about dot-delimited access.
303 
304  :param name:
305  :return: the value for the given name.
306  """
307  warnings.warn_explicit("Deprecated. Use []", DeprecationWarning)
308  return self[name]
309 
310  def setValue(self, name, value):
311  """Set the value for a parameter name/key. See class notes about dot-delimited access.
312 
313  :param name:
314  :return: None
315  """
316  warnings.warn("Deprecated. Use []", DeprecationWarning)
317  self[name] = value
318 
319  def mergeDefaults(self, other):
320  """For any keys in other that are not present in self, sets that key and its value into self.
321 
322  :param other: another Policy
323  :return: None
324  """
325  warnings.warn("Deprecated. Use .merge()", DeprecationWarning)
326  self.merge(other)
327 
328  def exists(self, key):
329  """Query if a key exists in this Policy
330 
331  :param key:
332  :return: True if the key exists, else false.
333  """
334  warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning)
335  return key in self
336 
337  def getString(self, key):
338  """Get the string value of a key.
339 
340  :param key:
341  :return: the value for key
342  """
343  warnings.warn("Deprecated. Use []", DeprecationWarning)
344  return str(self[key])
345 
346  def getBool(self, key):
347  """Get the value of a key.
348 
349  :param key:
350  :return: the value for key
351  """
352  warnings.warn("Deprecated. Use []", DeprecationWarning)
353  return bool(self[key])
354 
355  def getPolicy(self, key):
356  """Get a subpolicy.
357 
358  :param key:
359  :return:
360  """
361  warnings.warn("Deprecated. Use []", DeprecationWarning)
362  return self[key]
363 
364  def getStringArray(self, key):
365  """Get a value as an array. May contain one or more elements.
366 
367  :param key:
368  :return:
369  """
370  warnings.warn("Deprecated. Use asArray()", DeprecationWarning)
371  val = self.get(key)
372  if isinstance(val, basestring):
373  val = [val]
374  elif not isinstance(val, collections.Container):
375  val = [val]
376  return val
377 
378  def __lt__(self, other):
379  if isinstance(other, Policy):
380  other = other.data
381  return self.data < other
382 
383  def __le__(self, other):
384  if isinstance(other, Policy):
385  other = other.data
386  return self.data <= other
387 
388  def __eq__(self, other):
389  if isinstance(other, Policy):
390  other = other.data
391  return self.data == other
392 
393  def __ne__(self, other):
394  if isinstance(other, Policy):
395  other = other.data
396  return self.data != other
397 
398  def __gt__(self, other):
399  if isinstance(other, Policy):
400  other = other.data
401  return self.data > other
402 
403  def __ge__(self, other):
404  if isinstance(other, Policy):
405  other = other.data
406  return self.data >= other
407 
408  #######
409  # i/o #
410 
411  def dump(self, output):
412  """Writes the policy to a yaml stream.
413 
414  :param stream:
415  :return:
416  """
417  # First a set of known keys is handled and written to the stream in a specific order for readability.
418  # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to
419  # the stream.
420  data = copy.copy(self.data)
421  keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera',
422  'exposures', 'calibrations', 'datasets']
423  for key in keys:
424  try:
425  yaml.dump({key: data.pop(key)}, output, default_flow_style=False)
426  output.write('\n')
427  except KeyError:
428  pass
429  if data:
430  yaml.dump(data, output, default_flow_style=False)
431 
432  def dumpToFile(self, path):
433  """Writes the policy to a file.
434 
435  :param path:
436  :return:
437  """
438  with open(path, 'w') as f:
439  self.dump(f)