Coverage for python/lsst/daf/persistence/policy.py: 19%
189 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-23 02:30 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-23 02:30 -0700
1#!/usr/bin/env python
3#
4# LSST Data Management System
5# Copyright 2015 LSST Corporation.
6#
7# This product includes software developed by the
8# LSST Project (http://www.lsst.org/).
9#
10# This program is free software: you can redistribute it and/or modify
11# it under the terms of the GNU General Public License as published by
12# the Free Software Foundation, either version 3 of the License, or
13# (at your option) any later version.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the LSST License Statement and
21# the GNU General Public License along with this program. If not,
22# see <http://www.lsstcorp.org/LegalNotices/>.
23#
24import collections
25import collections.abc
26import copy
27import os
28import warnings
29import yaml
31import lsst.utils
33from yaml.representer import Representer
34yaml.add_representer(collections.defaultdict, Representer.represent_dict)
37# UserDict and yaml have defined metaclasses and Python 3 does not allow multiple
38# inheritance of classes with distinct metaclasses. We therefore have to
39# create a new baseclass that Policy can inherit from.
40class _PolicyMeta(type(collections.UserDict), type(yaml.YAMLObject)):
41 pass
44class _PolicyBase(collections.UserDict, yaml.YAMLObject, metaclass=_PolicyMeta):
45 pass
48class Policy(_PolicyBase):
49 """Policy implements a datatype that is used by Butler for configuration parameters.
50 It is essentially a dict with key/value pairs, including nested dicts (as values). In fact, it can be
51 initialized with a dict. The only caveat is that keys may NOT contain dots ('.'). This is explained next:
52 Policy extends the dict api so that hierarchical values may be accessed with dot-delimited notiation.
53 That is, foo.getValue('a.b.c') is the same as foo['a']['b']['c'] is the same as foo['a.b.c'], and either
54 of these syntaxes may be used.
56 Storage format supported:
57 - yaml: read and write is supported.
58 """
60 def __init__(self, other=None):
61 """Initialize the Policy. Other can be used to initialize the Policy in a variety of ways:
62 other (string) Treated as a path to a policy file on disk. Must end with '.yaml'.
63 other (Policy) Copies the other Policy's values into this one.
64 other (dict) Copies the values from the dict into this Policy.
65 """
66 collections.UserDict.__init__(self)
68 if other is None:
69 return
71 if isinstance(other, collections.abc.Mapping):
72 self.update(other)
73 elif isinstance(other, Policy):
74 self.data = copy.deepcopy(other.data)
75 elif isinstance(other, str):
76 # if other is a string, assume it is a file path.
77 self.__initFromFile(other)
78 else:
79 # if the policy specified by other could not be loaded raise a runtime error.
80 raise RuntimeError("A Policy could not be loaded from other:%s" % other)
82 def ppprint(self):
83 """helper function for debugging, prints a policy out in a readable way in the debugger.
85 use: pdb> print myPolicyObject.pprint()
86 :return: a prettyprint formatted string representing the policy
87 """
88 import pprint
89 return pprint.pformat(self.data, indent=2, width=1)
91 def __repr__(self):
92 return self.data.__repr__()
94 def __initFromFile(self, path):
95 """Load a file from path. If path is a list, will pick one to use, according to order specified
96 by extensionPreference.
98 :param path: string or list of strings, to a persisted policy file.
99 :param extensionPreference: the order in which to try to open files. Will use the first one that
100 succeeds.
101 :return:
102 """
103 if path.endswith('yaml'):
104 self.__initFromYamlFile(path)
105 else:
106 raise RuntimeError("Unhandled policy file type:%s" % path)
108 def __initFromYamlFile(self, path):
109 """Opens a file at a given path and attempts to load it in from yaml.
111 :param path:
112 :return:
113 """
114 with open(path, 'r') as f:
115 self.__initFromYaml(f)
117 def __initFromYaml(self, stream):
118 """Loads a YAML policy from any readable stream that contains one.
120 :param stream:
121 :return:
122 """
123 # will raise yaml.YAMLError if there is an error loading the file.
124 try:
125 # PyYAML >=5.1 prefers a different loader
126 loader = yaml.UnsafeLoader
127 except AttributeError:
128 loader = yaml.Loader
129 self.data = yaml.load(stream, Loader=loader)
130 return self
132 def __getitem__(self, name):
133 data = self.data
134 for key in name.split('.'):
135 if data is None:
136 return None
137 if key in data:
138 data = data[key]
139 else:
140 return None
141 if isinstance(data, collections.abc.Mapping):
142 data = Policy(data)
143 return data
145 def __setitem__(self, name, value):
146 if isinstance(value, collections.abc.Mapping):
147 keys = name.split('.')
148 d = {}
149 cur = d
150 for key in keys[0:-1]:
151 cur[key] = {}
152 cur = cur[key]
153 cur[keys[-1]] = value
154 self.update(d)
155 data = self.data
156 keys = name.split('.')
157 for key in keys[0:-1]:
158 data = data.setdefault(key, {})
159 data[keys[-1]] = value
161 def __contains__(self, key):
162 d = self.data
163 keys = key.split('.')
164 for k in keys[0:-1]:
165 if k in d:
166 d = d[k]
167 else:
168 return False
169 return keys[-1] in d
171 @staticmethod
172 def defaultPolicyFile(productName, fileName, relativePath=None):
173 """Get the path to a default policy file.
175 Determines a directory for the product specified by productName. Then Concatenates
176 productDir/relativePath/fileName (or productDir/fileName if relativePath is None) to find the path
177 to the default Policy file
179 @param productName (string) The name of the product that the default policy is installed as part of
180 @param fileName (string) The name of the policy file. Can also include a path to the file relative to
181 the directory where the product is installed.
182 @param relativePath (string) The relative path from the directior where the product is installed to
183 the location where the file (or the path to the file) is found. If None
184 (default), the fileName argument is relative to the installation
185 directory.
186 """
187 basePath = lsst.utils.getPackageDir(productName)
188 if not basePath:
189 raise RuntimeError("No product installed for productName: %s" % basePath)
190 if relativePath is not None:
191 basePath = os.path.join(basePath, relativePath)
192 fullFilePath = os.path.join(basePath, fileName)
193 return fullFilePath
195 def update(self, other):
196 """Like dict.update, but will add or modify keys in nested dicts, instead of overwriting the nested
197 dict entirely.
199 For example, for the given code:
200 foo = {'a': {'b': 1}}
201 foo.update({'a': {'c': 2}})
203 If foo is a dict, then after the update foo == {'a': {'c': 2}}
204 But if foo is a Policy, then after the update foo == {'a': {'b': 1, 'c': 2}}
205 """
206 def doUpdate(d, u):
207 for k, v in u.items():
208 if isinstance(d, collections.abc.Mapping):
209 if isinstance(v, collections.abc.Mapping):
210 r = doUpdate(d.get(k, {}), v)
211 d[k] = r
212 else:
213 d[k] = u[k]
214 else:
215 d = {k: u[k]}
216 return d
217 doUpdate(self.data, other)
219 def merge(self, other):
220 """Like Policy.update, but will add keys & values from other that DO NOT EXIST in self. Keys and
221 values that already exist in self will NOT be overwritten.
223 :param other:
224 :return:
225 """
226 otherCopy = copy.deepcopy(other)
227 otherCopy.update(self)
228 self.data = otherCopy.data
230 def names(self, topLevelOnly=False):
231 """Get the dot-delimited name of all the keys in the hierarchy.
232 NOTE: this is different than the built-in method dict.keys, which will return only the first level
233 keys.
234 """
235 if topLevelOnly:
236 return list(self.keys())
238 def getKeys(d, keys, base):
239 for key in d:
240 val = d[key]
241 levelKey = base + '.' + key if base is not None else key
242 keys.append(levelKey)
243 if isinstance(val, collections.abc.Mapping):
244 getKeys(val, keys, levelKey)
245 keys = []
246 getKeys(self.data, keys, None)
247 return keys
249 def asArray(self, name):
250 """Get a value as an array. May contain one or more elements.
252 :param key:
253 :return:
254 """
255 val = self.get(name)
256 if isinstance(val, str):
257 val = [val]
258 elif not isinstance(val, collections.abc.Container):
259 val = [val]
260 return val
262 # Deprecated methods that mimic pex_policy api.
263 # These are supported (for now), but callers should use the dict api.
265 def getValue(self, name):
266 """Get the value for a parameter name/key. See class notes about dot-delimited access.
268 :param name:
269 :return: the value for the given name.
270 """
271 warnings.warn_explicit("Deprecated. Use []", DeprecationWarning)
272 return self[name]
274 def setValue(self, name, value):
275 """Set the value for a parameter name/key. See class notes about dot-delimited access.
277 :param name:
278 :return: None
279 """
280 warnings.warn("Deprecated. Use []", DeprecationWarning)
281 self[name] = value
283 def mergeDefaults(self, other):
284 """For any keys in other that are not present in self, sets that key and its value into self.
286 :param other: another Policy
287 :return: None
288 """
289 warnings.warn("Deprecated. Use .merge()", DeprecationWarning)
290 self.merge(other)
292 def exists(self, key):
293 """Query if a key exists in this Policy
295 :param key:
296 :return: True if the key exists, else false.
297 """
298 warnings.warn("Deprecated. Use 'key in object'", DeprecationWarning)
299 return key in self
301 def getString(self, key):
302 """Get the string value of a key.
304 :param key:
305 :return: the value for key
306 """
307 warnings.warn("Deprecated. Use []", DeprecationWarning)
308 return str(self[key])
310 def getBool(self, key):
311 """Get the value of a key.
313 :param key:
314 :return: the value for key
315 """
316 warnings.warn("Deprecated. Use []", DeprecationWarning)
317 return bool(self[key])
319 def getPolicy(self, key):
320 """Get a subpolicy.
322 :param key:
323 :return:
324 """
325 warnings.warn("Deprecated. Use []", DeprecationWarning)
326 return self[key]
328 def getStringArray(self, key):
329 """Get a value as an array. May contain one or more elements.
331 :param key:
332 :return:
333 """
334 warnings.warn("Deprecated. Use asArray()", DeprecationWarning)
335 val = self.get(key)
336 if isinstance(val, str):
337 val = [val]
338 elif not isinstance(val, collections.abc.Container):
339 val = [val]
340 return val
342 def __lt__(self, other):
343 if isinstance(other, Policy):
344 other = other.data
345 return self.data < other
347 def __le__(self, other):
348 if isinstance(other, Policy):
349 other = other.data
350 return self.data <= other
352 def __eq__(self, other):
353 if isinstance(other, Policy):
354 other = other.data
355 return self.data == other
357 def __ne__(self, other):
358 if isinstance(other, Policy):
359 other = other.data
360 return self.data != other
362 def __gt__(self, other):
363 if isinstance(other, Policy):
364 other = other.data
365 return self.data > other
367 def __ge__(self, other):
368 if isinstance(other, Policy):
369 other = other.data
370 return self.data >= other
372 #######
373 # i/o #
375 def dump(self, output):
376 """Writes the policy to a yaml stream.
378 :param stream:
379 :return:
380 """
381 # First a set of known keys is handled and written to the stream in a specific order for readability.
382 # After the expected/ordered keys are weritten to the stream the remainder of the keys are written to
383 # the stream.
384 data = copy.copy(self.data)
385 keys = ['defects', 'needCalibRegistry', 'levels', 'defaultLevel', 'defaultSubLevels', 'camera',
386 'exposures', 'calibrations', 'datasets']
387 for key in keys:
388 try:
389 yaml.safe_dump({key: data.pop(key)}, output, default_flow_style=False)
390 output.write('\n')
391 except KeyError:
392 pass
393 if data:
394 yaml.safe_dump(data, output, default_flow_style=False)
396 def dumpToFile(self, path):
397 """Writes the policy to a file.
399 :param path:
400 :return:
401 """
402 with open(path, 'w') as f:
403 self.dump(f)