Coverage for python/lsst/ctrl/bps/bps_config.py: 12%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Configuration class that adds order to searching sections for value,
23expands environment variables and other config variables.
24"""
26__all__ = ["BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"]
29from os.path import expandvars
30import logging
31import copy
32import string
33import re
34from importlib.resources import path as resources_path
35import inflection
37from lsst.daf.butler.core.config import Config
39from . import etc
41_LOG = logging.getLogger(__name__)
43BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "bps_defined"]
45# Need a string that won't be a valid default value
46# to indicate whether default was defined for search.
47# And None is a valid default value.
48_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__"
51class BpsFormatter(string.Formatter):
52 """String formatter class that allows BPS config search options.
53 """
54 def get_field(self, field_name, args, kwargs):
55 _, val = args[0].search(field_name, opt=args[1])
56 return val, field_name
58 def get_value(self, key, args, kwargs):
59 _, val = args[0].search(key, opt=args[1])
60 return val
63class BpsConfig(Config):
64 """Contains the configuration for a BPS submission.
66 Parameters
67 ----------
68 other : `str`, `dict`, `Config`, `BpsConfig`
69 Path to a yaml file or a dict/Config/BpsConfig containing configuration
70 to copy.
71 search_order : `list` [`str`], optional
72 Root section names in the order in which they should be searched.
73 """
74 def __init__(self, other, search_order=None):
75 # In BPS config, the same setting can be defined multiple times in
76 # different sections. The sections are search in a pre-defined
77 # order. Hence, a value which is found first effectively overrides
78 # values in later sections, if any. To achieve this goal,
79 # the special methods __getitem__ and __contains__ were redefined to
80 # use a custom search function internally. For this reason we can't
81 # use super().__init__(other) as the super class defines its own
82 # __getitem__ which is utilized during the initialization process (
83 # e.g. in expressions like self[<key>]). However, this function will
84 # be overridden by the one defined here, in the subclass. Instead
85 # we just initialize internal data structures and populate them
86 # using the inherited update() method which does not rely on super
87 # class __getitem__ method.
88 super().__init__()
90 if isinstance(other, str):
91 # First load default config from ctrl_bps, then override with
92 # user config.
93 with resources_path(etc, "bps_defaults.yaml") as bps_defaults:
94 tmp_config = Config(str(bps_defaults))
95 user_config = Config(other)
96 tmp_config.update(user_config)
97 other = tmp_config
98 if search_order is None:
99 search_order = BPS_SEARCH_ORDER
101 try:
102 config = Config(other)
103 except RuntimeError:
104 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}")
105 self.update(config)
107 if isinstance(other, BpsConfig):
108 self.search_order = copy.deepcopy(other.search_order)
109 self.formatter = copy.deepcopy(other.formatter)
110 else:
111 if search_order is None:
112 search_order = []
113 self.search_order = search_order
114 self.formatter = BpsFormatter()
116 # Make sure search sections exist
117 for key in self.search_order:
118 if not Config.__contains__(self, key):
119 self[key] = {}
121 def copy(self):
122 """Make a copy of config.
124 Returns
125 -------
126 copy : `lsst.ctrl.bps.BpsConfig`
127 A duplicate of itself.
128 """
129 return BpsConfig(self)
131 def __getitem__(self, name):
132 """Return the value from the config for the given name.
134 Parameters
135 ----------
136 name : `str`
137 Key to look for in config
139 Returns
140 -------
141 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
142 Value from config if found.
143 """
144 _, val = self.search(name, {})
146 return val
148 def __contains__(self, name):
149 """Check whether name is in config.
151 Parameters
152 ----------
153 name : `str`
154 Key to look for in config.
156 Returns
157 -------
158 found : `bool`
159 Whether name was in config or not.
160 """
161 found, _ = self.search(name, {})
162 return found
164 @staticmethod
165 def _search_casing(sect, key):
166 # Until have more robust handling of key casing at config creation
167 # time, try checking here for different key casing.
168 found = False
169 value = ""
171 _LOG.debug("_search_casing: sect=%s key=%s", sect, key)
172 if Config.__contains__(sect, key):
173 found = True
174 value = Config.__getitem__(sect, key)
175 elif '_' in key:
176 newkey = inflection.camelize(key, False)
177 _LOG.debug("_search_casing: newkey=%s", newkey)
178 if Config.__contains__(sect, newkey):
179 found = True
180 value = Config.__getitem__(sect, newkey)
181 else: # try converting camel to snake
182 newkey = inflection.underscore(key)
183 _LOG.debug("_search_casing: newkey=%s", newkey)
184 if Config.__contains__(sect, newkey):
185 found = True
186 value = Config.__getitem__(sect, newkey)
187 else: # Try all lower case
188 newkey = key.lower()
189 _LOG.debug("_search_casing: newkey=%s", newkey)
190 if Config.__contains__(sect, newkey):
191 found = True
192 value = Config.__getitem__(sect, newkey)
194 return found, value
196 def search(self, key, opt=None):
197 """Search for key using given opt following hierarchy rules.
199 Search hierarchy rules: current values, a given search object, and
200 search order of config sections.
202 Parameters
203 ----------
204 key : `str`
205 Key to look for in config.
206 opt : `dict` [`str`, `Any`], optional
207 Options dictionary to use while searching. All are optional.
209 ``"curvals"``
210 Means to pass in values for search order key
211 (curr_<sectname>) or variable replacements.
212 (`dict`, optional)
213 ``"default"``
214 Value to return if not found. (`Any`, optional)
215 ``"replaceEnvVars"``
216 If search result is string, whether to replace environment
217 variables inside it with special placeholder (<ENV:name>).
218 By default set to False. (`bool`)
219 ``"expandEnvVars"``
220 If search result is string, whether to replace environment
221 variables inside it with current environment value.
222 By default set to False. (`bool`)
223 ``"replaceVars"``
224 If search result is string, whether to replace variables
225 inside it. By default set to True. (`bool`)
226 ``"required"``
227 If replacing variables, whether to raise exception if
228 variable is undefined. By default set to False. (`bool`)
230 Returns
231 -------
232 found : `bool`
233 Whether name was in config or not.
234 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
235 Value from config if found.
236 """
237 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
239 if opt is None:
240 opt = {}
242 found = False
243 value = ""
245 # start with stored current values
246 curvals = None
247 if Config.__contains__(self, "current"):
248 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
249 else:
250 curvals = {}
252 # override with current values passed into function if given
253 if "curvals" in opt:
254 for ckey, cval in list(opt["curvals"].items()):
255 _LOG.debug("using specified curval %s = %s", ckey, cval)
256 curvals[ckey] = cval
258 _LOG.debug("curvals = %s", curvals)
260 # There's a problem with the searchobj being a BpsConfig
261 # and its handling of __getitem__. Until that part of
262 # BpsConfig is rewritten, force the searchobj to a Config.
263 if "searchobj" in opt:
264 opt["searchobj"] = Config(opt["searchobj"])
266 if key in curvals:
267 _LOG.debug("found %s in curvals", key)
268 found = True
269 value = curvals[key]
270 elif "searchobj" in opt and key in opt["searchobj"]:
271 found = True
272 value = opt["searchobj"][key]
273 else:
274 for sect in self.search_order:
275 if Config.__contains__(self, sect):
276 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
277 search_sect = Config.__getitem__(self, sect)
278 if "curr_" + sect in curvals:
279 currkey = curvals["curr_" + sect]
280 _LOG.debug("currkey for section %s = %s", sect, currkey)
281 if Config.__contains__(search_sect, currkey):
282 search_sect = Config.__getitem__(search_sect, currkey)
284 found, value = self._search_casing(search_sect, key)
285 if found:
286 break
288 # lastly check root values
289 if not found:
290 _LOG.debug("Searching root section for key '%s'", key)
291 found, value = self._search_casing(self, key)
292 _LOG.debug(" root found=%s, value='%s'", found, value)
294 if not found and "default" in opt:
295 value = opt["default"]
296 found = True # ????
298 if not found and opt.get("required", False):
299 print(f"\n\nError: search for {key} failed")
300 print("\tcurrent = ", self.get("current"))
301 print("\topt = ", opt)
302 print("\tcurvals = ", curvals)
303 print("\n\n")
304 raise KeyError(f"Error: Search failed {key}")
306 _LOG.debug("found=%s, value=%s", found, value)
308 _LOG.debug("opt=%s %s", opt, type(opt))
309 if found and isinstance(value, str):
310 if opt.get("expandEnvVars", True):
311 _LOG.debug("before format=%s", value)
312 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value)
313 value = expandvars(value)
314 elif opt.get("replaceEnvVars", False):
315 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value)
316 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value)
318 if opt.get("replaceVars", True):
319 # default only applies to original search key
320 # Instead of doing deep copies of opt (especially with
321 # the recursive calls), temporarily remove default value
322 # and put it back.
323 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE)
325 # Temporarily replace any env vars so formatter doesn't try to
326 # replace them.
327 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value)
329 value = self.formatter.format(value, self, opt)
331 # Replace any temporary env place holders.
332 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value)
334 # if default was originally in opt
335 if default != _NO_SEARCH_DEFAULT_VALUE:
336 opt["default"] = default
338 _LOG.debug("after format=%s", value)
340 if found and isinstance(value, Config):
341 value = BpsConfig(value)
343 return found, value