Coverage for python/lsst/pipe/base/configOverrides.py: 21%
90 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 10:21 +0000
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-11 10:21 +0000
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module which defines ConfigOverrides class and related methods.
23"""
25__all__ = ["ConfigOverrides"]
27import ast
28import inspect
29from enum import Enum
30from operator import attrgetter
32from lsst.resources import ResourcePath
34from ._instrument import Instrument
36OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument")
39class ConfigExpressionParser(ast.NodeVisitor):
40 """An expression parser that will be used to transform configuration
41 strings supplied from the command line or a pipeline into a python
42 object.
44 This is roughly equivalent to ast.literal_parser, but with the ability to
45 transform strings that are valid variable names into the value associated
46 with the name. Variables that should be considered valid are supplied to
47 the constructor as a dictionary that maps a string to its corresponding
48 value.
50 This class in an internal implementation detail, and should not be exposed
51 outside this module.
53 Parameters
54 ----------
55 namespace : `dict` of `str` to variable
56 This is a mapping of strings corresponding to variable names, to the
57 object that is associated with that name
58 """
60 def __init__(self, namespace):
61 self.variables = namespace
63 def visit_Name(self, node):
64 """This method gets called when the parser has determined a node
65 corresponds to a variable name.
66 """
67 # If the id (name) of the variable is in the dictionary of valid names,
68 # load and return the corresponding variable.
69 if node.id in self.variables:
70 return self.variables[node.id]
71 # If the node does not correspond to a valid variable, turn the name
72 # into a string, as the user likely intended it as such.
73 return f"{node.id}"
75 def visit_List(self, node):
76 """This method is visited if the node is a list. Constructs a list out
77 of the sub nodes.
78 """
79 return [self.visit(elm) for elm in node.elts]
81 def visit_Tuple(self, node):
82 """This method is visited if the node is a tuple. Constructs a list out
83 of the sub nodes, and then turns it into a tuple.
84 """
85 return tuple(self.visit_List(node))
87 def visit_Constant(self, node):
88 """This method is visited if the node is a constant"""
89 return node.value
91 def visit_Dict(self, node):
92 """This method is visited if the node is a dict. It builds a dict out
93 of the component nodes.
94 """
95 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)}
97 def visit_Set(self, node):
98 """This method is visited if the node is a set. It builds a set out
99 of the component nodes.
100 """
101 return {self.visit(el) for el in node.elts}
103 def visit_UnaryOp(self, node):
104 """This method is visited if the node is a unary operator. Currently
105 The only operator we support is the negative (-) operator, all others
106 are passed to generic_visit method.
107 """
108 if isinstance(node.op, ast.USub):
109 value = self.visit(node.operand)
110 return -1 * value
111 self.generic_visit(node)
113 def generic_visit(self, node):
114 """This method is called for all other node types. It will just raise
115 a value error, because this is a type of expression that we do not
116 support.
117 """
118 raise ValueError("Unable to parse string into literal expression")
121class ConfigOverrides:
122 """Defines a set of overrides to be applied to a task config.
124 Overrides for task configuration need to be applied by activator when
125 creating task instances. This class represents an ordered set of such
126 overrides which activator receives from some source (e.g. command line
127 or some other configuration).
129 Methods
130 ----------
131 addFileOverride(filename)
132 Add overrides from a specified file.
133 addValueOverride(field, value)
134 Add override for a specific field.
135 applyTo(config)
136 Apply all overrides to a `config` instance.
138 Notes
139 -----
140 Serialization support for this class may be needed, will add later if
141 necessary.
142 """
144 def __init__(self):
145 self._overrides = []
147 def addFileOverride(self, filename):
148 """Add overrides from a specified file.
150 Parameters
151 ----------
152 filename : convertible to `ResourcePath`
153 Path or URI to the override file. All URI schemes supported by
154 `ResourcePath` are supported.
155 """
156 self._overrides.append((OverrideTypes.File, ResourcePath(filename)))
158 def addValueOverride(self, field, value):
159 """Add override for a specific field.
161 This method is not very type-safe as it is designed to support
162 use cases where input is given as string, e.g. command line
163 activators. If `value` has a string type and setting of the field
164 fails with `TypeError` the we'll attempt `eval()` the value and
165 set the field with that value instead.
167 Parameters
168 ----------
169 field : str
170 Fully-qualified field name.
171 value :
172 Value to be given to a filed.
173 """
174 self._overrides.append((OverrideTypes.Value, (field, value)))
176 def addPythonOverride(self, python_snippet: str) -> None:
177 """Add Overrides by running a snippit of python code against a config.
179 Parameters
180 ----------
181 python_snippet: str
182 A string which is valid python code to be executed. This is done
183 with config as the only local accessible value.
184 """
185 self._overrides.append((OverrideTypes.Python, python_snippet))
187 def addInstrumentOverride(self, instrument: Instrument, task_name: str) -> None:
188 """Apply any overrides that an instrument has for a task
190 Parameters
191 ----------
192 instrument: `Instrument`
193 An instrument instance which will apply configs
194 task_name: str
195 The _DefaultName of a task associated with a config, used to look
196 up overrides from the instrument.
197 """
198 self._overrides.append((OverrideTypes.Instrument, (instrument, task_name)))
200 def _parser(self, value, configParser):
201 try:
202 value = configParser.visit(ast.parse(value, mode="eval").body)
203 except Exception:
204 # This probably means it is a specific user string such as a URI.
205 # Let the value return as a string to attempt to continue to
206 # process as a string, another error will be raised in downstream
207 # code if that assumption is wrong
208 pass
210 return value
212 def applyTo(self, config):
213 """Apply all overrides to a task configuration object.
215 Parameters
216 ----------
217 config : `pex.Config`
218 Configuration to apply to; modified in place.
220 Raises
221 ------
222 `Exception` is raised if operations on configuration object fail.
223 """
224 # Look up a stack of variables people may be using when setting
225 # configs. Create a dictionary that will be used akin to a namespace
226 # for the duration of this function.
227 vars = {}
228 # pull in the variables that are declared in module scope of the config
229 mod = inspect.getmodule(config)
230 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")})
231 # put the supplied config in the variables dictionary
232 vars["config"] = config
234 # Create a parser for config expressions that may be strings
235 configParser = ConfigExpressionParser(namespace=vars)
237 for otype, override in self._overrides:
238 if otype is OverrideTypes.File:
239 with override.open("r") as buffer:
240 config.loadFromStream(buffer, filename=override.ospath)
241 elif otype is OverrideTypes.Value:
242 field, value = override
243 if isinstance(value, str):
244 value = self._parser(value, configParser)
245 # checking for dicts and lists here is needed because {} and []
246 # are valid yaml syntax so they get converted before hitting
247 # this method, so we must parse the elements.
248 #
249 # The same override would remain a string if specified on the
250 # command line, and is handled above.
251 if isinstance(value, dict):
252 new = {}
253 for k, v in value.items():
254 if isinstance(v, str):
255 new[k] = self._parser(v, configParser)
256 else:
257 new[k] = v
258 value = new
259 elif isinstance(value, list):
260 new = []
261 for v in value:
262 if isinstance(v, str):
263 new.append(self._parser(v, configParser))
264 else:
265 new.append(v)
266 value = new
267 # The field might be a string corresponding to a attribute
268 # hierarchy, attempt to split off the last field which
269 # will then be set.
270 parent, *child = field.rsplit(".", maxsplit=1)
271 if child:
272 # This branch means there was a hierarchy, get the
273 # field to set, and look up the sub config for which
274 # it is to be set
275 finalField = child[0]
276 tmpConfig = attrgetter(parent)(config)
277 else:
278 # There is no hierarchy, the config is the base config
279 # and the field is exactly what was passed in
280 finalField = parent
281 tmpConfig = config
282 # set the specified config
283 setattr(tmpConfig, finalField, value)
285 elif otype is OverrideTypes.Python:
286 # exec python string with the context of all vars known. This
287 # both lets people use a var they know about (maybe a bit
288 # dangerous, but not really more so than arbitrary python exec,
289 # and there are so many other places to worry about security
290 # before we start changing this) and any imports that are done
291 # in a python block will be put into this scope. This means
292 # other config setting branches can make use of these
293 # variables.
294 exec(override, None, vars)
295 elif otype is OverrideTypes.Instrument:
296 instrument, name = override
297 instrument.applyConfigOverrides(name, config)