Coverage for python/lsst/ctrl/bps/bps_config.py: 15%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Configuration class that adds order to searching sections for value,
23expands environment variables and other config variables.
24"""
26__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"]
29import copy
30import logging
31import re
32import string
33from importlib import resources
34from os.path import expandvars
36from lsst.daf.butler.core.config import Config
38from . import etc
40_LOG = logging.getLogger(__name__)
42# Using lsst.daf.butler.Config to resolve possible includes.
43with resources.path(etc, "bps_defaults.yaml") as path:
44 BPS_DEFAULTS = Config(str(path)).toDict()
46BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "bps_defined"]
48# Need a string that won't be a valid default value
49# to indicate whether default was defined for search.
50# And None is a valid default value.
51_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__"
54class BpsFormatter(string.Formatter):
55 """String formatter class that allows BPS config search options.
56 """
57 def get_field(self, field_name, args, kwargs):
58 _, val = args[0].search(field_name, opt=args[1])
59 return val, field_name
61 def get_value(self, key, args, kwargs):
62 _, val = args[0].search(key, opt=args[1])
63 return val
66class BpsConfig(Config):
67 """Contains the configuration for a BPS submission.
69 Parameters
70 ----------
71 other : `str`, `dict`, `Config`, `BpsConfig`
72 Path to a yaml file or a dict/Config/BpsConfig containing configuration
73 to copy.
74 search_order : `list` [`str`], optional
75 Root section names in the order in which they should be searched.
76 """
77 def __init__(self, other, search_order=None):
78 # In BPS config, the same setting can be defined multiple times in
79 # different sections. The sections are search in a pre-defined
80 # order. Hence, a value which is found first effectively overrides
81 # values in later sections, if any. To achieve this goal,
82 # the special methods __getitem__ and __contains__ were redefined to
83 # use a custom search function internally. For this reason we can't
84 # use super().__init__(other) as the super class defines its own
85 # __getitem__ which is utilized during the initialization process (
86 # e.g. in expressions like self[<key>]). However, this function will
87 # be overridden by the one defined here, in the subclass. Instead
88 # we just initialize internal data structures and populate them
89 # using the inherited update() method which does not rely on super
90 # class __getitem__ method.
91 super().__init__()
93 if isinstance(other, str):
94 # First load default config from ctrl_bps, then override with
95 # user config.
96 tmp_config = Config(BPS_DEFAULTS)
97 user_config = Config(other)
98 tmp_config.update(user_config)
99 other = tmp_config
100 if search_order is None:
101 search_order = BPS_SEARCH_ORDER
103 try:
104 config = Config(other)
105 except RuntimeError:
106 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}")
107 self.update(config)
109 if isinstance(other, BpsConfig):
110 self.search_order = copy.deepcopy(other.search_order)
111 self.formatter = copy.deepcopy(other.formatter)
112 else:
113 if search_order is None:
114 search_order = []
115 self.search_order = search_order
116 self.formatter = BpsFormatter()
118 # Make sure search sections exist
119 for key in self.search_order:
120 if not Config.__contains__(self, key):
121 self[key] = {}
123 def copy(self):
124 """Make a copy of config.
126 Returns
127 -------
128 copy : `lsst.ctrl.bps.BpsConfig`
129 A duplicate of itself.
130 """
131 return BpsConfig(self)
133 def __getitem__(self, name):
134 """Return the value from the config for the given name.
136 Parameters
137 ----------
138 name : `str`
139 Key to look for in config
141 Returns
142 -------
143 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
144 Value from config if found.
145 """
146 _, val = self.search(name, {})
148 return val
150 def __contains__(self, name):
151 """Check whether name is in config.
153 Parameters
154 ----------
155 name : `str`
156 Key to look for in config.
158 Returns
159 -------
160 found : `bool`
161 Whether name was in config or not.
162 """
163 found, _ = self.search(name, {})
164 return found
166 def search(self, key, opt=None):
167 """Search for key using given opt following hierarchy rules.
169 Search hierarchy rules: current values, a given search object, and
170 search order of config sections.
172 Parameters
173 ----------
174 key : `str`
175 Key to look for in config.
176 opt : `dict` [`str`, `Any`], optional
177 Options dictionary to use while searching. All are optional.
179 ``"curvals"``
180 Means to pass in values for search order key
181 (curr_<sectname>) or variable replacements.
182 (`dict`, optional)
183 ``"default"``
184 Value to return if not found. (`Any`, optional)
185 ``"replaceEnvVars"``
186 If search result is string, whether to replace environment
187 variables inside it with special placeholder (<ENV:name>).
188 By default set to False. (`bool`)
189 ``"expandEnvVars"``
190 If search result is string, whether to replace environment
191 variables inside it with current environment value.
192 By default set to False. (`bool`)
193 ``"replaceVars"``
194 If search result is string, whether to replace variables
195 inside it. By default set to True. (`bool`)
196 ``"required"``
197 If replacing variables, whether to raise exception if
198 variable is undefined. By default set to False. (`bool`)
200 Returns
201 -------
202 found : `bool`
203 Whether name was in config or not.
204 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
205 Value from config if found.
206 """
207 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
209 if opt is None:
210 opt = {}
212 found = False
213 value = ""
215 # start with stored current values
216 curvals = None
217 if Config.__contains__(self, "current"):
218 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
219 else:
220 curvals = {}
222 # override with current values passed into function if given
223 if "curvals" in opt:
224 for ckey, cval in list(opt["curvals"].items()):
225 _LOG.debug("using specified curval %s = %s", ckey, cval)
226 curvals[ckey] = cval
228 _LOG.debug("curvals = %s", curvals)
230 # There's a problem with the searchobj being a BpsConfig
231 # and its handling of __getitem__. Until that part of
232 # BpsConfig is rewritten, force the searchobj to a Config.
233 if "searchobj" in opt:
234 opt["searchobj"] = Config(opt["searchobj"])
236 if key in curvals:
237 _LOG.debug("found %s in curvals", key)
238 found = True
239 value = curvals[key]
240 elif "searchobj" in opt and key in opt["searchobj"]:
241 found = True
242 value = opt["searchobj"][key]
243 else:
244 for sect in self.search_order:
245 if Config.__contains__(self, sect):
246 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
247 search_sect = Config.__getitem__(self, sect)
248 if "curr_" + sect in curvals:
249 currkey = curvals["curr_" + sect]
250 _LOG.debug("currkey for section %s = %s", sect, currkey)
251 if Config.__contains__(search_sect, currkey):
252 search_sect = Config.__getitem__(search_sect, currkey)
254 _LOG.debug("%s %s", key, search_sect)
255 if Config.__contains__(search_sect, key):
256 found = True
257 value = Config.__getitem__(search_sect, key)
258 break
259 else:
260 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key)
262 # lastly check root values
263 if not found:
264 _LOG.debug("Searching root section for key '%s'", key)
265 if Config.__contains__(self, key):
266 found = True
267 value = Config.__getitem__(self, key)
268 _LOG.debug("root value='%s'", value)
270 if not found and "default" in opt:
271 value = opt["default"]
272 found = True # ????
274 if not found and opt.get("required", False):
275 print(f"\n\nError: search for {key} failed")
276 print("\tcurrent = ", self.get("current"))
277 print("\topt = ", opt)
278 print("\tcurvals = ", curvals)
279 print("\n\n")
280 raise KeyError(f"Error: Search failed {key}")
282 _LOG.debug("found=%s, value=%s", found, value)
284 _LOG.debug("opt=%s %s", opt, type(opt))
285 if found and isinstance(value, str):
286 if opt.get("expandEnvVars", True):
287 _LOG.debug("before format=%s", value)
288 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value)
289 value = expandvars(value)
290 elif opt.get("replaceEnvVars", False):
291 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value)
292 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value)
294 if opt.get("replaceVars", True):
295 # default only applies to original search key
296 # Instead of doing deep copies of opt (especially with
297 # the recursive calls), temporarily remove default value
298 # and put it back.
299 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE)
301 # Temporarily replace any env vars so formatter doesn't try to
302 # replace them.
303 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value)
305 value = self.formatter.format(value, self, opt)
307 # Replace any temporary env place holders.
308 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value)
310 # if default was originally in opt
311 if default != _NO_SEARCH_DEFAULT_VALUE:
312 opt["default"] = default
314 _LOG.debug("after format=%s", value)
316 if found and isinstance(value, Config):
317 value = BpsConfig(value)
319 return found, value