Coverage for python/lsst/ctrl/bps/bps_config.py: 12%
132 statements
« prev ^ index » next coverage.py v7.3.0, created at 2023-09-02 09:44 +0000
« prev ^ index » next coverage.py v7.3.0, created at 2023-09-02 09:44 +0000
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Configuration class that adds order to searching sections for value,
23expands environment variables and other config variables.
24"""
26__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"]
29import copy
30import logging
31import re
32import string
33from os.path import expandvars
35from lsst.daf.butler import Config
36from lsst.resources import ResourcePath
38_LOG = logging.getLogger(__name__)
40# Using lsst.daf.butler.Config to resolve possible includes.
41BPS_DEFAULTS = Config(ResourcePath("resource://lsst.ctrl.bps/etc/bps_defaults.yaml")).toDict()
43BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "cloud", "bps_defined"]
45# Need a string that won't be a valid default value
46# to indicate whether default was defined for search.
47# And None is a valid default value.
48_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__"
51class BpsFormatter(string.Formatter):
52 """String formatter class that allows BPS config search options."""
54 def get_field(self, field_name, args, kwargs):
55 _, val = args[0].search(field_name, opt=args[1])
56 return val, field_name
58 def get_value(self, key, args, kwargs):
59 _, val = args[0].search(key, opt=args[1])
60 return val
63class BpsConfig(Config):
64 """Contains the configuration for a BPS submission.
66 Parameters
67 ----------
68 other : `str`, `dict`, `~lsst.daf.butler.Config`, `BpsConfig`
69 Path to a yaml file or a dict/Config/BpsConfig containing configuration
70 to copy.
71 search_order : `list` [`str`], optional
72 Root section names in the order in which they should be searched.
73 """
75 def __init__(self, other, search_order=None):
76 # In BPS config, the same setting can be defined multiple times in
77 # different sections. The sections are search in a pre-defined
78 # order. Hence, a value which is found first effectively overrides
79 # values in later sections, if any. To achieve this goal,
80 # the special methods __getitem__ and __contains__ were redefined to
81 # use a custom search function internally. For this reason we can't
82 # use super().__init__(other) as the super class defines its own
83 # __getitem__ which is utilized during the initialization process (
84 # e.g. in expressions like self[<key>]). However, this function will
85 # be overridden by the one defined here, in the subclass. Instead
86 # we just initialize internal data structures and populate them
87 # using the inherited update() method which does not rely on super
88 # class __getitem__ method.
89 super().__init__()
91 if isinstance(other, str):
92 # First load default config from ctrl_bps, then override with
93 # user config.
94 tmp_config = Config(BPS_DEFAULTS)
95 user_config = Config(other)
96 tmp_config.update(user_config)
97 other = tmp_config
98 if search_order is None:
99 search_order = BPS_SEARCH_ORDER
101 try:
102 config = Config(other)
103 except RuntimeError:
104 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}")
105 self.update(config)
107 if isinstance(other, BpsConfig):
108 self.search_order = copy.deepcopy(other.search_order)
109 self.formatter = copy.deepcopy(other.formatter)
110 else:
111 if search_order is None:
112 search_order = []
113 self.search_order = search_order
114 self.formatter = BpsFormatter()
116 # Make sure search sections exist
117 for key in self.search_order:
118 if not Config.__contains__(self, key):
119 self[key] = {}
121 def copy(self):
122 """Make a copy of config.
124 Returns
125 -------
126 copy : `lsst.ctrl.bps.BpsConfig`
127 A duplicate of itself.
128 """
129 return BpsConfig(self)
131 def get(self, key, default=""):
132 """Return the value for key if key is in the config, else default.
134 If default is not given, it defaults to an empty string.
136 Parameters
137 ----------
138 key : `str`
139 Key to look for in config.
140 default : Any, optional
141 Default value to return if the key is not in the config.
143 Returns
144 -------
145 val : Any
146 Value from config if found, default otherwise.
148 Notes
149 -----
150 The provided default value (an empty string) was chosen to maintain
151 the internal consistency with other methods of the class.
152 """
153 _, val = self.search(key, opt={"default": default})
154 return val
156 def __getitem__(self, name):
157 """Return the value from the config for the given name.
159 Parameters
160 ----------
161 name : `str`
162 Key to look for in config
164 Returns
165 -------
166 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
167 Value from config if found.
168 """
169 _, val = self.search(name, {})
171 return val
173 def __contains__(self, name):
174 """Check whether name is in config.
176 Parameters
177 ----------
178 name : `str`
179 Key to look for in config.
181 Returns
182 -------
183 found : `bool`
184 Whether name was in config or not.
185 """
186 found, _ = self.search(name, {})
187 return found
189 def search(self, key, opt=None):
190 """Search for key using given opt following hierarchy rules.
192 Search hierarchy rules: current values, a given search object, and
193 search order of config sections.
195 Parameters
196 ----------
197 key : `str`
198 Key to look for in config.
199 opt : `dict` [`str`, `Any`], optional
200 Options dictionary to use while searching. All are optional.
202 ``"curvals"``
203 Means to pass in values for search order key
204 (curr_<sectname>) or variable replacements.
205 (`dict`, optional)
206 ``"default"``
207 Value to return if not found. (`Any`, optional)
208 ``"replaceEnvVars"``
209 If search result is string, whether to replace environment
210 variables inside it with special placeholder (<ENV:name>).
211 By default set to False. (`bool`)
212 ``"expandEnvVars"``
213 If search result is string, whether to replace environment
214 variables inside it with current environment value.
215 By default set to False. (`bool`)
216 ``"replaceVars"``
217 If search result is string, whether to replace variables
218 inside it. By default set to True. (`bool`)
219 ``"required"``
220 If replacing variables, whether to raise exception if
221 variable is undefined. By default set to False. (`bool`)
223 Returns
224 -------
225 found : `bool`
226 Whether name was in config or not.
227 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
228 Value from config if found.
229 """
230 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
232 if opt is None:
233 opt = {}
235 found = False
236 value = ""
238 # start with stored current values
239 curvals = None
240 if Config.__contains__(self, "current"):
241 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
242 else:
243 curvals = {}
245 # override with current values passed into function if given
246 if "curvals" in opt:
247 for ckey, cval in list(opt["curvals"].items()):
248 _LOG.debug("using specified curval %s = %s", ckey, cval)
249 curvals[ckey] = cval
251 _LOG.debug("curvals = %s", curvals)
253 # There's a problem with the searchobj being a BpsConfig
254 # and its handling of __getitem__. Until that part of
255 # BpsConfig is rewritten, force the searchobj to a Config.
256 if "searchobj" in opt:
257 opt["searchobj"] = Config(opt["searchobj"])
259 if key in curvals:
260 _LOG.debug("found %s in curvals", key)
261 found = True
262 value = curvals[key]
263 elif "searchobj" in opt and key in opt["searchobj"]:
264 found = True
265 value = opt["searchobj"][key]
266 else:
267 for sect in self.search_order:
268 if Config.__contains__(self, sect):
269 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
270 search_sect = Config.__getitem__(self, sect)
271 if "curr_" + sect in curvals:
272 currkey = curvals["curr_" + sect]
273 _LOG.debug("currkey for section %s = %s", sect, currkey)
274 if Config.__contains__(search_sect, currkey):
275 search_sect = Config.__getitem__(search_sect, currkey)
277 _LOG.debug("%s %s", key, search_sect)
278 if Config.__contains__(search_sect, key):
279 found = True
280 value = Config.__getitem__(search_sect, key)
281 break
282 else:
283 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key)
285 # lastly check root values
286 if not found:
287 _LOG.debug("Searching root section for key '%s'", key)
288 if Config.__contains__(self, key):
289 found = True
290 value = Config.__getitem__(self, key)
291 _LOG.debug("root value='%s'", value)
293 if not found and "default" in opt:
294 value = opt["default"]
295 found = True # ????
297 if not found and opt.get("required", False):
298 print(f"\n\nError: search for {key} failed")
299 print("\tcurrent = ", self.get("current"))
300 print("\topt = ", opt)
301 print("\tcurvals = ", curvals)
302 print("\n\n")
303 raise KeyError(f"Error: Search failed {key}")
305 _LOG.debug("found=%s, value=%s", found, value)
307 _LOG.debug("opt=%s %s", opt, type(opt))
308 if found and isinstance(value, str):
309 if opt.get("expandEnvVars", True):
310 _LOG.debug("before format=%s", value)
311 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value)
312 value = expandvars(value)
313 elif opt.get("replaceEnvVars", False):
314 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value)
315 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value)
317 if opt.get("replaceVars", True):
318 # default only applies to original search key
319 # Instead of doing deep copies of opt (especially with
320 # the recursive calls), temporarily remove default value
321 # and put it back.
322 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE)
324 # Temporarily replace any env vars so formatter doesn't try to
325 # replace them.
326 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value)
328 value = self.formatter.format(value, self, opt)
330 # Replace any temporary env place holders.
331 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value)
333 # if default was originally in opt
334 if default != _NO_SEARCH_DEFAULT_VALUE:
335 opt["default"] = default
337 _LOG.debug("after format=%s", value)
339 if found and isinstance(value, Config):
340 value = BpsConfig(value)
342 return found, value