Coverage for python/lsst/pipe/base/configOverrides.py: 21%
90 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:10 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-14 02:10 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module which defines ConfigOverrides class and related methods.
23"""
25__all__ = ["ConfigOverrides"]
27import ast
28import inspect
29from enum import Enum
30from operator import attrgetter
32from lsst.resources import ResourcePath
34from ._instrument import Instrument
36OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument")
39class ConfigExpressionParser(ast.NodeVisitor):
40 """An expression parser that will be used to transform configuration
41 strings supplied from the command line or a pipeline into a python
42 object.
44 This is roughly equivalent to ast.literal_parser, but with the ability to
45 transform strings that are valid variable names into the value associated
46 with the name. Variables that should be considered valid are supplied to
47 the constructor as a dictionary that maps a string to its corresponding
48 value.
50 This class in an internal implementation detail, and should not be exposed
51 outside this module.
53 Parameters
54 ----------
55 namespace : `dict` of `str` to variable
56 This is a mapping of strings corresponding to variable names, to the
57 object that is associated with that name
58 """
60 def __init__(self, namespace):
61 self.variables = namespace
63 def visit_Name(self, node):
64 """Handle a node corresponding to a variable name."""
65 # If the id (name) of the variable is in the dictionary of valid names,
66 # load and return the corresponding variable.
67 if node.id in self.variables:
68 return self.variables[node.id]
69 # If the node does not correspond to a valid variable, turn the name
70 # into a string, as the user likely intended it as such.
71 return f"{node.id}"
73 def visit_List(self, node):
74 """Build a list out of the sub nodes when a list node is
75 encountered.
76 """
77 return [self.visit(elm) for elm in node.elts]
79 def visit_Tuple(self, node):
80 """Build a list out of the sub nodes and then turn it into a
81 tuple.
82 """
83 return tuple(self.visit_List(node))
85 def visit_Constant(self, node):
86 """Return constant from node."""
87 return node.value
89 def visit_Dict(self, node):
90 """Build dict out of component nodes if dict node encountered."""
91 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)}
93 def visit_Set(self, node):
94 """Build set out of node is set encountered."""
95 return {self.visit(el) for el in node.elts}
97 def visit_UnaryOp(self, node):
98 """Handle unary operators.
100 This method is visited if the node is a unary operator. Currently
101 The only operator we support is the negative (-) operator, all others
102 are passed to generic_visit method.
103 """
104 if isinstance(node.op, ast.USub):
105 value = self.visit(node.operand)
106 return -1 * value
107 self.generic_visit(node)
109 def generic_visit(self, node):
110 """Handle other node types.
112 This method is called for all other node types. It will just raise
113 a value error, because this is a type of expression that we do not
114 support.
115 """
116 raise ValueError("Unable to parse string into literal expression")
119class ConfigOverrides:
120 """Defines a set of overrides to be applied to a task config.
122 Overrides for task configuration need to be applied by activator when
123 creating task instances. This class represents an ordered set of such
124 overrides which activator receives from some source (e.g. command line
125 or some other configuration).
127 Methods
128 -------
129 addFileOverride(filename)
130 Add overrides from a specified file.
131 addValueOverride(field, value)
132 Add override for a specific field.
133 applyTo(config)
134 Apply all overrides to a `config` instance.
136 Notes
137 -----
138 Serialization support for this class may be needed, will add later if
139 necessary.
140 """
142 def __init__(self):
143 self._overrides = []
145 def addFileOverride(self, filename):
146 """Add overrides from a specified file.
148 Parameters
149 ----------
150 filename : convertible to `~lsst.resources.ResourcePath`
151 Path or URI to the override file. All URI schemes supported by
152 `~lsst.resources.ResourcePath` are supported.
153 """
154 self._overrides.append((OverrideTypes.File, ResourcePath(filename)))
156 def addValueOverride(self, field, value):
157 """Add override for a specific field.
159 This method is not very type-safe as it is designed to support
160 use cases where input is given as string, e.g. command line
161 activators. If `value` has a string type and setting of the field
162 fails with `TypeError` the we'll attempt `eval()` the value and
163 set the field with that value instead.
165 Parameters
166 ----------
167 field : str
168 Fully-qualified field name.
169 value :
170 Value to be given to a filed.
171 """
172 self._overrides.append((OverrideTypes.Value, (field, value)))
174 def addPythonOverride(self, python_snippet: str) -> None:
175 """Add Overrides by running a snippit of python code against a config.
177 Parameters
178 ----------
179 python_snippet: str
180 A string which is valid python code to be executed. This is done
181 with config as the only local accessible value.
182 """
183 self._overrides.append((OverrideTypes.Python, python_snippet))
185 def addInstrumentOverride(self, instrument: Instrument, task_name: str) -> None:
186 """Apply any overrides that an instrument has for a task
188 Parameters
189 ----------
190 instrument: `Instrument`
191 An instrument instance which will apply configs
192 task_name: str
193 The _DefaultName of a task associated with a config, used to look
194 up overrides from the instrument.
195 """
196 self._overrides.append((OverrideTypes.Instrument, (instrument, task_name)))
198 def _parser(self, value, configParser):
199 try:
200 value = configParser.visit(ast.parse(value, mode="eval").body)
201 except Exception:
202 # This probably means it is a specific user string such as a URI.
203 # Let the value return as a string to attempt to continue to
204 # process as a string, another error will be raised in downstream
205 # code if that assumption is wrong
206 pass
208 return value
210 def applyTo(self, config):
211 """Apply all overrides to a task configuration object.
213 Parameters
214 ----------
215 config : `pex.Config`
216 Configuration to apply to; modified in place.
218 Raises
219 ------
220 `Exception` is raised if operations on configuration object fail.
221 """
222 # Look up a stack of variables people may be using when setting
223 # configs. Create a dictionary that will be used akin to a namespace
224 # for the duration of this function.
225 vars = {}
226 # pull in the variables that are declared in module scope of the config
227 mod = inspect.getmodule(config)
228 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")})
229 # put the supplied config in the variables dictionary
230 vars["config"] = config
232 # Create a parser for config expressions that may be strings
233 configParser = ConfigExpressionParser(namespace=vars)
235 for otype, override in self._overrides:
236 if otype is OverrideTypes.File:
237 with override.open("r") as buffer:
238 config.loadFromStream(buffer, filename=override.ospath)
239 elif otype is OverrideTypes.Value:
240 field, value = override
241 if isinstance(value, str):
242 value = self._parser(value, configParser)
243 # checking for dicts and lists here is needed because {} and []
244 # are valid yaml syntax so they get converted before hitting
245 # this method, so we must parse the elements.
246 #
247 # The same override would remain a string if specified on the
248 # command line, and is handled above.
249 if isinstance(value, dict):
250 new = {}
251 for k, v in value.items():
252 if isinstance(v, str):
253 new[k] = self._parser(v, configParser)
254 else:
255 new[k] = v
256 value = new
257 elif isinstance(value, list):
258 new = []
259 for v in value:
260 if isinstance(v, str):
261 new.append(self._parser(v, configParser))
262 else:
263 new.append(v)
264 value = new
265 # The field might be a string corresponding to a attribute
266 # hierarchy, attempt to split off the last field which
267 # will then be set.
268 parent, *child = field.rsplit(".", maxsplit=1)
269 if child:
270 # This branch means there was a hierarchy, get the
271 # field to set, and look up the sub config for which
272 # it is to be set
273 finalField = child[0]
274 tmpConfig = attrgetter(parent)(config)
275 else:
276 # There is no hierarchy, the config is the base config
277 # and the field is exactly what was passed in
278 finalField = parent
279 tmpConfig = config
280 # set the specified config
281 setattr(tmpConfig, finalField, value)
283 elif otype is OverrideTypes.Python:
284 # exec python string with the context of all vars known. This
285 # both lets people use a var they know about (maybe a bit
286 # dangerous, but not really more so than arbitrary python exec,
287 # and there are so many other places to worry about security
288 # before we start changing this) and any imports that are done
289 # in a python block will be put into this scope. This means
290 # other config setting branches can make use of these
291 # variables.
292 exec(override, None, vars)
293 elif otype is OverrideTypes.Instrument:
294 instrument, name = override
295 instrument.applyConfigOverrides(name, config)