Coverage for python/lsst/ctrl/bps/bps_config.py : 12%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Configuration class that adds order to searching sections for value,
24expands environment variables and other config variables
25"""
27from os.path import expandvars
28import logging
29import copy
30import string
32from lsst.daf.butler.core.config import Config
34_LOG = logging.getLogger()
37class BpsFormatter(string.Formatter):
38 """String formatter class that allows BPS config
39 search options
40 """
41 def get_field(self, field_name, args, kwargs):
42 _, val = args[0].search(field_name, opt=args[1])
43 return (val, field_name)
45 def get_value(self, key, args, kwargs):
46 _, val = args[0].search(key, opt=args[1])
47 return val
50class BpsConfig(Config):
51 """Contains the configuration for a BPS submission.
53 Parameters
54 ----------
55 other: `str`, `dict`, `Config`, `BpsConfig`
56 Path to a yaml file or a dict/Config/BpsConfig containing configuration
57 to copy.
58 """
59 def __init__(self, other, search_order=None):
60 # In BPS config, the same setting can be defined multiple times in
61 # different sections. The sections are search in a pre-defined
62 # order. Hence, a value which is found first effectively overrides
63 # values in later sections, if any. To achieve this goal,
64 # the special methods __getitem__ and __contains__ were redefined to
65 # use a custom search function internally. For this reason we can't
66 # use super().__init__(other) as the super class defines its own
67 # __getitem__ which is utilized during the initialization process (
68 # e.g. in expressions like self[<key>]). However, this function will
69 # be overridden by the one defined here, in the subclass. Instead
70 # we just initialize internal data structures and populate them
71 # using the inherited update() method which does not rely on super
72 # class __getitem__ method.
73 super().__init__()
74 try:
75 config = Config(other)
76 except RuntimeError:
77 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other)
78 self.update(config)
80 if isinstance(other, BpsConfig):
81 self.search_order = copy.deepcopy(other.search_order)
82 self.formatter = copy.deepcopy(other.formatter)
83 else:
84 if search_order is None:
85 search_order = []
86 self.search_order = search_order
87 self.formatter = BpsFormatter()
89 def copy(self):
90 """Makes a copy of config
92 Returns
93 -------
94 copy: `BpsConfig`
95 A duplicate of itself
96 """
97 return BpsConfig(self)
99 def __getitem__(self, name):
100 """Returns the value from the config for the given name
102 Parameters
103 ----------
104 name: `str`
105 Key to look for in config
107 Returns
108 -------
109 val: `str`, `int`, `BPSConfig`, ...
110 Value from config if found
111 """
112 _, val = self.search(name, {})
114 return val
116 def __contains__(self, name):
117 """Checks whether name is in config
119 Parameters
120 ----------
121 name: `str`
122 Key to look for in config
124 Returns
125 -------
126 found: `bool`
127 Whether name was in config or not
128 """
129 found, _ = self.search(name, {})
130 return found
132 def search(self, key, opt=None):
133 """Searches for key using given opt following hierarchy rules.
135 Search hierarchy rules: current values, a given search object, and
136 search order of config sections.
138 Parameters
139 ----------
140 key: `str`
141 Key to look for in config.
142 opt: `dict`, optional
143 Options dictionary to use while searching. All are optional.
145 ``"curvals"``
146 Means to pass in values for search order key
147 (curr_<sectname>) or variable replacements.
148 (`dict`, optional)
149 ``"default"``
150 Value to return if not found. (`Any`, optional)
151 ``"replaceVars"``
152 If search result is string, whether to replace variables
153 inside it. By default set to True. (`bool`)
154 ``"required"``
155 If replacing variables, whether to raise exception if
156 variable is undefined. By default set to False. (`bool`)
158 Returns
159 -------
160 found: `bool`
161 Whether name was in config or not
162 value: `str`, `int`, `BpsConfig`, ...
163 Value from config if found
164 """
165 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
167 if opt is None:
168 opt = {}
170 found = False
171 value = ""
173 # start with stored current values
174 curvals = None
175 if Config.__contains__(self, "current"):
176 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
177 else:
178 curvals = {}
180 # override with current values passed into function if given
181 if "curvals" in opt:
182 for ckey, cval in list(opt["curvals"].items()):
183 _LOG.debug("using specified curval %s = %s", ckey, cval)
184 curvals[ckey] = cval
186 _LOG.debug("curvals = %s", curvals)
188 if key in curvals:
189 _LOG.debug("found %s in curvals", key)
190 found = True
191 value = curvals[key]
192 elif "searchobj" in opt and key in opt["searchobj"]:
193 found = True
194 value = opt["searchobj"][key]
195 else:
196 for sect in self.search_order:
197 if Config.__contains__(self, sect):
198 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
199 search_sect = Config.__getitem__(self, sect)
200 if "curr_" + sect in curvals:
201 currkey = curvals["curr_" + sect]
202 _LOG.debug("currkey for section %s = %s", sect, currkey)
203 # search_sect = Config.__getitem__(search_sect, currkey)
204 if Config.__contains__(search_sect, currkey):
205 search_sect = Config.__getitem__(search_sect, currkey)
207 _LOG.debug("%s %s", key, search_sect)
208 if Config.__contains__(search_sect, key):
209 found = True
210 value = Config.__getitem__(search_sect, key)
211 break
212 else:
213 _LOG.warning("Missing search section '%s' while searching for '%s'", sect, key)
215 # lastly check root values
216 if not found:
217 _LOG.debug("Searching root section for key '%s'", key)
218 if Config.__contains__(self, key):
219 found = True
220 value = Config.__getitem__(self, key)
221 _LOG.debug("root value='%s'", value)
223 if not found and "default" in opt:
224 value = opt["default"]
225 found = True # ????
227 if not found and opt.get("required", False):
228 print("\n\nError: search for %s failed" % (key))
229 print("\tcurrent = ", self.get("current"))
230 print("\topt = ", opt)
231 print("\tcurvals = ", curvals)
232 print("\n\n")
233 raise KeyError("Error: Search failed (%s)" % key)
235 _LOG.debug("found=%s, value=%s", found, value)
237 _LOG.debug("opt=%s %s", opt, type(opt))
238 if found and isinstance(value, str) and opt.get("replaceVars", True):
239 _LOG.debug("before format=%s", value)
240 value = expandvars(value) # must replace env vars before calling format
241 value = self.formatter.format(value, self, opt)
242 _LOG.debug("after format=%s", value)
244 if found and isinstance(value, Config):
245 value = BpsConfig(value)
247 return found, value