Coverage for python/lsst/ctrl/bps/bps_config.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""Configuration class that adds order to searching sections for value,
23expands environment variables and other config variables.
24"""
26__all__ = ["BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"]
29from os.path import expandvars
30import logging
31import copy
32import string
33import re
35from lsst.daf.butler.core.config import Config
38_LOG = logging.getLogger(__name__)
40BPS_SEARCH_ORDER = ["payload", "pipetask", "site", "bps_defined"]
43class BpsFormatter(string.Formatter):
44 """String formatter class that allows BPS config search options.
45 """
46 def get_field(self, field_name, args, kwargs):
47 _, val = args[0].search(field_name, opt=args[1])
48 return val, field_name
50 def get_value(self, key, args, kwargs):
51 _, val = args[0].search(key, opt=args[1])
52 return val
55class BpsConfig(Config):
56 """Contains the configuration for a BPS submission.
58 Parameters
59 ----------
60 other : `str`, `dict`, `Config`, `BpsConfig`
61 Path to a yaml file or a dict/Config/BpsConfig containing configuration
62 to copy.
63 search_order : `list` [`str`], optional
64 Root section names in the order in which they should be searched.
65 """
66 def __init__(self, other, search_order=None):
67 # In BPS config, the same setting can be defined multiple times in
68 # different sections. The sections are search in a pre-defined
69 # order. Hence, a value which is found first effectively overrides
70 # values in later sections, if any. To achieve this goal,
71 # the special methods __getitem__ and __contains__ were redefined to
72 # use a custom search function internally. For this reason we can't
73 # use super().__init__(other) as the super class defines its own
74 # __getitem__ which is utilized during the initialization process (
75 # e.g. in expressions like self[<key>]). However, this function will
76 # be overridden by the one defined here, in the subclass. Instead
77 # we just initialize internal data structures and populate them
78 # using the inherited update() method which does not rely on super
79 # class __getitem__ method.
80 super().__init__()
81 try:
82 config = Config(other)
83 except RuntimeError:
84 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other)
85 self.update(config)
87 if isinstance(other, BpsConfig):
88 self.search_order = copy.deepcopy(other.search_order)
89 self.formatter = copy.deepcopy(other.formatter)
90 else:
91 if search_order is None:
92 search_order = []
93 self.search_order = search_order
94 self.formatter = BpsFormatter()
96 # Make sure search sections exist
97 for key in self.search_order:
98 if not Config.__contains__(self, key):
99 self[key] = {}
101 def copy(self):
102 """Make a copy of config.
104 Returns
105 -------
106 copy : `lsst.ctrl.bps.BpsConfig`
107 A duplicate of itself.
108 """
109 return BpsConfig(self)
111 def __getitem__(self, name):
112 """Return the value from the config for the given name.
114 Parameters
115 ----------
116 name : `str`
117 Key to look for in config
119 Returns
120 -------
121 val : `str`, `int`, `lsst.ctrl.bps.BPSConfig`, ...
122 Value from config if found.
123 """
124 _, val = self.search(name, {})
126 return val
128 def __contains__(self, name):
129 """Check whether name is in config.
131 Parameters
132 ----------
133 name : `str`
134 Key to look for in config.
136 Returns
137 -------
138 found : `bool`
139 Whether name was in config or not.
140 """
141 found, _ = self.search(name, {})
142 return found
144 def search(self, key, opt=None):
145 """Search for key using given opt following hierarchy rules.
147 Search hierarchy rules: current values, a given search object, and
148 search order of config sections.
150 Parameters
151 ----------
152 key : `str`
153 Key to look for in config.
154 opt : `dict` [`str`, `Any`], optional
155 Options dictionary to use while searching. All are optional.
157 ``"curvals"``
158 Means to pass in values for search order key
159 (curr_<sectname>) or variable replacements.
160 (`dict`, optional)
161 ``"default"``
162 Value to return if not found. (`Any`, optional)
163 ``"replaceEnvVars"``
164 If search result is string, whether to replace environment
165 variables inside it with special placeholder (<ENV:name>).
166 By default set to False. (`bool`)
167 ``"expandEnvVars"``
168 If search result is string, whether to replace environment
169 variables inside it with current environment value.
170 By default set to False. (`bool`)
171 ``"replaceVars"``
172 If search result is string, whether to replace variables
173 inside it. By default set to True. (`bool`)
174 ``"required"``
175 If replacing variables, whether to raise exception if
176 variable is undefined. By default set to False. (`bool`)
178 Returns
179 -------
180 found : `bool`
181 Whether name was in config or not.
182 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ...
183 Value from config if found.
184 """
185 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
187 if opt is None:
188 opt = {}
190 found = False
191 value = ""
193 # start with stored current values
194 curvals = None
195 if Config.__contains__(self, "current"):
196 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
197 else:
198 curvals = {}
200 # override with current values passed into function if given
201 if "curvals" in opt:
202 for ckey, cval in list(opt["curvals"].items()):
203 _LOG.debug("using specified curval %s = %s", ckey, cval)
204 curvals[ckey] = cval
206 _LOG.debug("curvals = %s", curvals)
208 if key in curvals:
209 _LOG.debug("found %s in curvals", key)
210 found = True
211 value = curvals[key]
212 elif "searchobj" in opt and key in opt["searchobj"]:
213 found = True
214 value = opt["searchobj"][key]
215 else:
216 for sect in self.search_order:
217 if Config.__contains__(self, sect):
218 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
219 search_sect = Config.__getitem__(self, sect)
220 if "curr_" + sect in curvals:
221 currkey = curvals["curr_" + sect]
222 _LOG.debug("currkey for section %s = %s", sect, currkey)
223 if Config.__contains__(search_sect, currkey):
224 search_sect = Config.__getitem__(search_sect, currkey)
226 _LOG.debug("%s %s", key, search_sect)
227 if Config.__contains__(search_sect, key):
228 found = True
229 value = Config.__getitem__(search_sect, key)
230 break
231 else:
232 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key)
234 # lastly check root values
235 if not found:
236 _LOG.debug("Searching root section for key '%s'", key)
237 if Config.__contains__(self, key):
238 found = True
239 value = Config.__getitem__(self, key)
240 _LOG.debug("root value='%s'", value)
242 if not found and "default" in opt:
243 value = opt["default"]
244 found = True # ????
246 if not found and opt.get("required", False):
247 print("\n\nError: search for %s failed" % (key))
248 print("\tcurrent = ", self.get("current"))
249 print("\topt = ", opt)
250 print("\tcurvals = ", curvals)
251 print("\n\n")
252 raise KeyError("Error: Search failed (%s)" % key)
254 _LOG.debug("found=%s, value=%s", found, value)
256 _LOG.debug("opt=%s %s", opt, type(opt))
257 if found and isinstance(value, str):
258 if opt.get("expandEnvVars", True):
259 _LOG.debug("before format=%s", value)
260 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value)
261 value = expandvars(value)
262 elif opt.get("replaceEnvVars", False):
263 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value)
264 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value)
266 if opt.get("replaceVars", True):
267 # default only applies to original search key
268 default = opt.pop("default", None)
270 # Temporarily replace any env vars so formatter doesn't try to
271 # replace them.
272 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value)
274 value = self.formatter.format(value, self, opt)
276 # Replace any temporary env place holders.
277 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value)
279 if default: # reset to avoid modifying dict parameter.
280 opt["default"] = default
282 _LOG.debug("after format=%s", value)
284 if found and isinstance(value, Config):
285 value = BpsConfig(value)
287 return found, value