Coverage for python/lsst/ctrl/bps/bps_config.py : 10%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Configuration class that adds order to searching sections for value,
24expands environment variables and other config variables
25"""
27from os.path import expandvars
28import logging
29import copy
30import string
31import re
33from lsst.daf.butler.core.config import Config
35_LOG = logging.getLogger(__name__)
38class BpsFormatter(string.Formatter):
39 """String formatter class that allows BPS config
40 search options
41 """
42 def get_field(self, field_name, args, kwargs):
43 _, val = args[0].search(field_name, opt=args[1])
44 return (val, field_name)
46 def get_value(self, key, args, kwargs):
47 _, val = args[0].search(key, opt=args[1])
48 return val
51class BpsConfig(Config):
52 """Contains the configuration for a BPS submission.
54 Parameters
55 ----------
56 other : `str`, `dict`, `Config`, `BpsConfig`
57 Path to a yaml file or a dict/Config/BpsConfig containing configuration
58 to copy.
59 search_order : `list` of `str`, optional
60 Root section names in the order in which they should be searched.
61 """
62 def __init__(self, other, search_order=None):
63 # In BPS config, the same setting can be defined multiple times in
64 # different sections. The sections are search in a pre-defined
65 # order. Hence, a value which is found first effectively overrides
66 # values in later sections, if any. To achieve this goal,
67 # the special methods __getitem__ and __contains__ were redefined to
68 # use a custom search function internally. For this reason we can't
69 # use super().__init__(other) as the super class defines its own
70 # __getitem__ which is utilized during the initialization process (
71 # e.g. in expressions like self[<key>]). However, this function will
72 # be overridden by the one defined here, in the subclass. Instead
73 # we just initialize internal data structures and populate them
74 # using the inherited update() method which does not rely on super
75 # class __getitem__ method.
76 super().__init__()
77 try:
78 config = Config(other)
79 except RuntimeError:
80 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other)
81 self.update(config)
83 if isinstance(other, BpsConfig):
84 self.search_order = copy.deepcopy(other.search_order)
85 self.formatter = copy.deepcopy(other.formatter)
86 else:
87 if search_order is None:
88 search_order = []
89 self.search_order = search_order
90 self.formatter = BpsFormatter()
92 # Make sure search sections exist
93 for key in self.search_order:
94 if not Config.__contains__(self, key):
95 self[key] = {}
97 def copy(self):
98 """Makes a copy of config
100 Returns
101 -------
102 copy : `~lsst.ctrl.bps.bps_config.BpsConfig`
103 A duplicate of itself
104 """
105 return BpsConfig(self)
107 def __getitem__(self, name):
108 """Returns the value from the config for the given name
110 Parameters
111 ----------
112 name : `str`
113 Key to look for in config
115 Returns
116 -------
117 val : `str`, `int`, `~lsst.ctrl.bps.bps_config.BPSConfig`, ...
118 Value from config if found
119 """
120 _, val = self.search(name, {})
122 return val
124 def __contains__(self, name):
125 """Checks whether name is in config.
127 Parameters
128 ----------
129 name : `str`
130 Key to look for in config.
132 Returns
133 -------
134 found : `bool`
135 Whether name was in config or not
136 """
137 found, _ = self.search(name, {})
138 return found
140 def search(self, key, opt=None):
141 """Searches for key using given opt following hierarchy rules.
143 Search hierarchy rules: current values, a given search object, and
144 search order of config sections.
146 Parameters
147 ----------
148 key : `str`
149 Key to look for in config.
150 opt : `dict`, optional
151 Options dictionary to use while searching. All are optional.
153 ``"curvals"``
154 Means to pass in values for search order key
155 (curr_<sectname>) or variable replacements.
156 (`dict`, optional)
157 ``"default"``
158 Value to return if not found. (`Any`, optional)
159 ``"replaceEnvVars"``
160 If search result is string, whether to replace environment
161 variables inside it with special placeholder (<ENV:name>).
162 By default set to False. (`bool`)
163 ``"expandEnvVars"``
164 If search result is string, whether to replace environment
165 variables inside it with current environment value.
166 By default set to False. (`bool`)
167 ``"replaceVars"``
168 If search result is string, whether to replace variables
169 inside it. By default set to True. (`bool`)
170 ``"required"``
171 If replacing variables, whether to raise exception if
172 variable is undefined. By default set to False. (`bool`)
174 Returns
175 -------
176 found : `bool`
177 Whether name was in config or not
178 value : `str`, `int`, `BpsConfig`, ...
179 Value from config if found
180 """
181 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
183 if opt is None:
184 opt = {}
186 found = False
187 value = ""
189 # start with stored current values
190 curvals = None
191 if Config.__contains__(self, "current"):
192 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
193 else:
194 curvals = {}
196 # override with current values passed into function if given
197 if "curvals" in opt:
198 for ckey, cval in list(opt["curvals"].items()):
199 _LOG.debug("using specified curval %s = %s", ckey, cval)
200 curvals[ckey] = cval
202 _LOG.debug("curvals = %s", curvals)
204 if key in curvals:
205 _LOG.debug("found %s in curvals", key)
206 found = True
207 value = curvals[key]
208 elif "searchobj" in opt and key in opt["searchobj"]:
209 found = True
210 value = opt["searchobj"][key]
211 else:
212 for sect in self.search_order:
213 if Config.__contains__(self, sect):
214 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
215 search_sect = Config.__getitem__(self, sect)
216 if "curr_" + sect in curvals:
217 currkey = curvals["curr_" + sect]
218 _LOG.debug("currkey for section %s = %s", sect, currkey)
219 # search_sect = Config.__getitem__(search_sect, currkey)
220 if Config.__contains__(search_sect, currkey):
221 search_sect = Config.__getitem__(search_sect, currkey)
223 _LOG.debug("%s %s", key, search_sect)
224 if Config.__contains__(search_sect, key):
225 found = True
226 value = Config.__getitem__(search_sect, key)
227 break
228 else:
229 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key)
231 # lastly check root values
232 if not found:
233 _LOG.debug("Searching root section for key '%s'", key)
234 if Config.__contains__(self, key):
235 found = True
236 value = Config.__getitem__(self, key)
237 _LOG.debug("root value='%s'", value)
239 if not found and "default" in opt:
240 value = opt["default"]
241 found = True # ????
243 if not found and opt.get("required", False):
244 print("\n\nError: search for %s failed" % (key))
245 print("\tcurrent = ", self.get("current"))
246 print("\topt = ", opt)
247 print("\tcurvals = ", curvals)
248 print("\n\n")
249 raise KeyError("Error: Search failed (%s)" % key)
251 _LOG.debug("found=%s, value=%s", found, value)
253 _LOG.debug("opt=%s %s", opt, type(opt))
254 if found and isinstance(value, str):
255 if opt.get("expandEnvVars", True):
256 _LOG.debug("before format=%s", value)
257 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value)
258 value = expandvars(value)
259 elif opt.get("replaceEnvVars", False):
260 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value)
261 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value)
263 if opt.get("replaceVars", True):
264 # default only applies to original search key
265 default = opt.pop("default", None)
267 # Temporarily replace any env vars so formatter doesn't try to replace them.
268 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value)
270 value = self.formatter.format(value, self, opt)
272 # Replace any temporary env place holders.
273 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value)
275 if default: # reset to avoid modifying dict parameter.
276 opt["default"] = default
278 _LOG.debug("after format=%s", value)
280 if found and isinstance(value, Config):
281 value = BpsConfig(value)
283 return found, value