22 """Module which defines ConfigOverrides class and related methods.
25 __all__ = [
"ConfigOverrides"]
28 from operator
import attrgetter
36 OverrideTypes = Enum(
"OverrideTypes",
"Value File Python Instrument")
40 """An expression parser that will be used to transform configuration
41 strings supplied from the command line or a pipeline into a python
44 This is roughly equivalent to ast.literal_parser, but with the ability to
45 transform strings that are valid variable names into the value associated
46 with the name. Variables that should be considered valid are supplied to
47 the constructor as a dictionary that maps a string to its corresponding
50 This class in an internal implementation detail, and should not be exposed
55 namespace : `dict` of `str` to variable
56 This is a mapping of strings corresponding to variable names, to the
57 object that is associated with that name
64 """This method gets called when the parser has determined a node
65 corresponds to a variable name.
76 """This method is visited if the node is a list. Constructs a list out
79 return [self.visit(elm)
for elm
in node.elts]
82 """This method is visited if the node is a tuple. Constructs a list out
83 of the sub nodes, and then turns it into a tuple.
88 """This method is visited if the node is a constant
93 """This method is visited if the node is a dict. It builds a dict out
94 of the component nodes.
96 return {self.visit(key): self.visit(value)
for key, value
in zip(node.keys, node.values)}
99 """This method is visited if the node is a set. It builds a set out
100 of the component nodes.
102 return {self.visit(el)
for el
in node.elts}
105 """This method is visited if the node is a unary operator. Currently
106 The only operator we support is the negative (-) operator, all others
107 are passed to generic_visit method.
109 if isinstance(node.op, ast.USub):
110 value = self.visit(node.operand)
115 """This method is called for all other node types. It will just raise
116 a value error, because this is a type of expression that we do not
119 raise ValueError(
"Unable to parse string into literal expression")
123 """Defines a set of overrides to be applied to a task config.
125 Overrides for task configuration need to be applied by activator when
126 creating task instances. This class represents an ordered set of such
127 overrides which activator receives from some source (e.g. command line
128 or some other configuration).
132 addFileOverride(filename)
133 Add overrides from a specified file.
134 addValueOverride(field, value)
135 Add override for a specific field.
137 Apply all overrides to a `config` instance.
141 Serialization support for this class may be needed, will add later if
149 """Add overrides from a specified file.
154 Path to the override file.
156 self.
_overrides_overrides.append((OverrideTypes.File, filename))
159 """Add override for a specific field.
161 This method is not very type-safe as it is designed to support
162 use cases where input is given as string, e.g. command line
163 activators. If `value` has a string type and setting of the field
164 fails with `TypeError` the we'll attempt `eval()` the value and
165 set the field with that value instead.
170 Fully-qualified field name.
172 Value to be given to a filed.
174 self.
_overrides_overrides.append((OverrideTypes.Value, (field, value)))
177 """Add Overrides by running a snippit of python code against a config.
182 A string which is valid python code to be executed. This is done
183 with config as the only local accessible value.
185 self.
_overrides_overrides.append((OverrideTypes.Python, python_snippet))
188 """Apply any overrides that an instrument has for a task
193 A string containing the fully qualified name of an instrument from
194 which configs should be loaded and applied
196 The _DefaultName of a task associated with a config, used to look
197 up overrides from the instrument.
199 instrument_lib = doImport(instrument)()
200 self.
_overrides_overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name)))
202 def _parser(self, value, configParser):
204 value = configParser.visit(ast.parse(value, mode=
'eval').body)
208 raise pexExceptions.RuntimeError(f
"Unable to parse `{value}' into a Python object")
from None
212 """Apply all overrides to a task configuration object.
216 config : `pex.Config`
220 `Exception` is raised if operations on configuration object fail.
227 mod = inspect.getmodule(config)
228 vars.update({k: v
for k, v
in mod.__dict__.items()
if not k.startswith(
"__")})
230 vars[
'config'] = config
235 for otype, override
in self.
_overrides_overrides:
236 if otype
is OverrideTypes.File:
237 config.load(override)
238 elif otype
is OverrideTypes.Value:
239 field, value = override
240 if isinstance(value, str):
241 value = self.
_parser_parser(value, configParser)
248 if isinstance(value, dict):
250 for k, v
in value.items():
251 if isinstance(v, str):
252 new[k] = self.
_parser_parser(v, configParser)
256 elif isinstance(value, list):
259 if isinstance(v, str):
260 new.append(self.
_parser_parser(v, configParser))
267 parent, *child = field.rsplit(
".", maxsplit=1)
272 finalField = child[0]
273 tmpConfig = attrgetter(parent)(config)
280 setattr(tmpConfig, finalField, value)
282 elif otype
is OverrideTypes.Python:
291 exec(override,
None, vars)
292 elif otype
is OverrideTypes.Instrument:
293 instrument, name = override
294 instrument.applyConfigOverrides(name, config)
def __init__(self, namespace)
def visit_Set(self, node)
def visit_Constant(self, node)
def generic_visit(self, node)
def visit_Dict(self, node)
def visit_Tuple(self, node)
def visit_Name(self, node)
def visit_List(self, node)
def visit_UnaryOp(self, node)
def addPythonOverride(self, str python_snippet)
def addInstrumentOverride(self, str instrument, str task_name)
def addFileOverride(self, filename)
def applyTo(self, config)
def addValueOverride(self, field, value)
def _parser(self, value, configParser)