Coverage for python/lsst/ctrl/bps/bps_config.py : 11%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ctrl_bps.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22"""
23Configuration class that adds order to searching sections for value,
24expands environment variables and other config variables
25"""
27from os.path import expandvars
28import logging
29import copy
30import string
32from lsst.daf.butler.core.config import Config
34_LOG = logging.getLogger(__name__)
37class BpsFormatter(string.Formatter):
38 """String formatter class that allows BPS config
39 search options
40 """
41 def get_field(self, field_name, args, kwargs):
42 _, val = args[0].search(field_name, opt=args[1])
43 return (val, field_name)
45 def get_value(self, key, args, kwargs):
46 _, val = args[0].search(key, opt=args[1])
47 return val
50class BpsConfig(Config):
51 """Contains the configuration for a BPS submission.
53 Parameters
54 ----------
55 other : `str`, `dict`, `Config`, `BpsConfig`
56 Path to a yaml file or a dict/Config/BpsConfig containing configuration
57 to copy.
58 search_order : `list` of `str`, optional
59 Root section names in the order in which they should be searched.
60 """
61 def __init__(self, other, search_order=None):
62 # In BPS config, the same setting can be defined multiple times in
63 # different sections. The sections are search in a pre-defined
64 # order. Hence, a value which is found first effectively overrides
65 # values in later sections, if any. To achieve this goal,
66 # the special methods __getitem__ and __contains__ were redefined to
67 # use a custom search function internally. For this reason we can't
68 # use super().__init__(other) as the super class defines its own
69 # __getitem__ which is utilized during the initialization process (
70 # e.g. in expressions like self[<key>]). However, this function will
71 # be overridden by the one defined here, in the subclass. Instead
72 # we just initialize internal data structures and populate them
73 # using the inherited update() method which does not rely on super
74 # class __getitem__ method.
75 super().__init__()
76 try:
77 config = Config(other)
78 except RuntimeError:
79 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other)
80 self.update(config)
82 if isinstance(other, BpsConfig):
83 self.search_order = copy.deepcopy(other.search_order)
84 self.formatter = copy.deepcopy(other.formatter)
85 else:
86 if search_order is None:
87 search_order = []
88 self.search_order = search_order
89 self.formatter = BpsFormatter()
91 # Make sure search sections exist
92 for key in self.search_order:
93 if not Config.__contains__(self, key):
94 self[key] = {}
96 def copy(self):
97 """Makes a copy of config
99 Returns
100 -------
101 copy : `~lsst.ctrl.bps.bps_config.BpsConfig`
102 A duplicate of itself
103 """
104 return BpsConfig(self)
106 def __getitem__(self, name):
107 """Returns the value from the config for the given name
109 Parameters
110 ----------
111 name : `str`
112 Key to look for in config
114 Returns
115 -------
116 val : `str`, `int`, `~lsst.ctrl.bps.bps_config.BPSConfig`, ...
117 Value from config if found
118 """
119 _, val = self.search(name, {})
121 return val
123 def __contains__(self, name):
124 """Checks whether name is in config.
126 Parameters
127 ----------
128 name : `str`
129 Key to look for in config.
131 Returns
132 -------
133 found : `bool`
134 Whether name was in config or not
135 """
136 found, _ = self.search(name, {})
137 return found
139 def search(self, key, opt=None):
140 """Searches for key using given opt following hierarchy rules.
142 Search hierarchy rules: current values, a given search object, and
143 search order of config sections.
145 Parameters
146 ----------
147 key : `str`
148 Key to look for in config.
149 opt : `dict`, optional
150 Options dictionary to use while searching. All are optional.
152 ``"curvals"``
153 Means to pass in values for search order key
154 (curr_<sectname>) or variable replacements.
155 (`dict`, optional)
156 ``"default"``
157 Value to return if not found. (`Any`, optional)
158 ``"replaceVars"``
159 If search result is string, whether to replace variables
160 inside it. By default set to True. (`bool`)
161 ``"required"``
162 If replacing variables, whether to raise exception if
163 variable is undefined. By default set to False. (`bool`)
165 Returns
166 -------
167 found : `bool`
168 Whether name was in config or not
169 value : `str`, `int`, `BpsConfig`, ...
170 Value from config if found
171 """
172 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt)
174 if opt is None:
175 opt = {}
177 found = False
178 value = ""
180 # start with stored current values
181 curvals = None
182 if Config.__contains__(self, "current"):
183 curvals = copy.deepcopy(Config.__getitem__(self, "current"))
184 else:
185 curvals = {}
187 # override with current values passed into function if given
188 if "curvals" in opt:
189 for ckey, cval in list(opt["curvals"].items()):
190 _LOG.debug("using specified curval %s = %s", ckey, cval)
191 curvals[ckey] = cval
193 _LOG.debug("curvals = %s", curvals)
195 if key in curvals:
196 _LOG.debug("found %s in curvals", key)
197 found = True
198 value = curvals[key]
199 elif "searchobj" in opt and key in opt["searchobj"]:
200 found = True
201 value = opt["searchobj"][key]
202 else:
203 for sect in self.search_order:
204 if Config.__contains__(self, sect):
205 _LOG.debug("Searching '%s' section for key '%s'", sect, key)
206 search_sect = Config.__getitem__(self, sect)
207 if "curr_" + sect in curvals:
208 currkey = curvals["curr_" + sect]
209 _LOG.debug("currkey for section %s = %s", sect, currkey)
210 # search_sect = Config.__getitem__(search_sect, currkey)
211 if Config.__contains__(search_sect, currkey):
212 search_sect = Config.__getitem__(search_sect, currkey)
214 _LOG.debug("%s %s", key, search_sect)
215 if Config.__contains__(search_sect, key):
216 found = True
217 value = Config.__getitem__(search_sect, key)
218 break
219 else:
220 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key)
222 # lastly check root values
223 if not found:
224 _LOG.debug("Searching root section for key '%s'", key)
225 if Config.__contains__(self, key):
226 found = True
227 value = Config.__getitem__(self, key)
228 _LOG.debug("root value='%s'", value)
230 if not found and "default" in opt:
231 value = opt["default"]
232 found = True # ????
234 if not found and opt.get("required", False):
235 print("\n\nError: search for %s failed" % (key))
236 print("\tcurrent = ", self.get("current"))
237 print("\topt = ", opt)
238 print("\tcurvals = ", curvals)
239 print("\n\n")
240 raise KeyError("Error: Search failed (%s)" % key)
242 _LOG.debug("found=%s, value=%s", found, value)
244 _LOG.debug("opt=%s %s", opt, type(opt))
245 if found and isinstance(value, str) and opt.get("replaceVars", True):
246 _LOG.debug("before format=%s", value)
247 value = expandvars(value) # must replace env vars before calling format
248 value = self.formatter.format(value, self, opt)
249 _LOG.debug("after format=%s", value)
251 if found and isinstance(value, Config):
252 value = BpsConfig(value)
254 return found, value