Coverage for python/lsst/pipe/base/configOverrides.py: 20%
92 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 16:29 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-13 16:29 -0700
1# This file is part of pipe_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22"""Module which defines ConfigOverrides class and related methods.
23"""
25__all__ = ["ConfigOverrides"]
27import ast
28import inspect
29from enum import Enum
30from operator import attrgetter
32from lsst.resources import ResourcePath
33from lsst.utils import doImportType
35OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument")
38class ConfigExpressionParser(ast.NodeVisitor):
39 """An expression parser that will be used to transform configuration
40 strings supplied from the command line or a pipeline into a python
41 object.
43 This is roughly equivalent to ast.literal_parser, but with the ability to
44 transform strings that are valid variable names into the value associated
45 with the name. Variables that should be considered valid are supplied to
46 the constructor as a dictionary that maps a string to its corresponding
47 value.
49 This class in an internal implementation detail, and should not be exposed
50 outside this module.
52 Parameters
53 ----------
54 namespace : `dict` of `str` to variable
55 This is a mapping of strings corresponding to variable names, to the
56 object that is associated with that name
57 """
59 def __init__(self, namespace):
60 self.variables = namespace
62 def visit_Name(self, node):
63 """This method gets called when the parser has determined a node
64 corresponds to a variable name.
65 """
66 # If the id (name) of the variable is in the dictionary of valid names,
67 # load and return the corresponding variable.
68 if node.id in self.variables:
69 return self.variables[node.id]
70 # If the node does not correspond to a valid variable, turn the name
71 # into a string, as the user likely intended it as such.
72 return f"{node.id}"
74 def visit_List(self, node):
75 """This method is visited if the node is a list. Constructs a list out
76 of the sub nodes.
77 """
78 return [self.visit(elm) for elm in node.elts]
80 def visit_Tuple(self, node):
81 """This method is visited if the node is a tuple. Constructs a list out
82 of the sub nodes, and then turns it into a tuple.
83 """
84 return tuple(self.visit_List(node))
86 def visit_Constant(self, node):
87 """This method is visited if the node is a constant"""
88 return node.value
90 def visit_Dict(self, node):
91 """This method is visited if the node is a dict. It builds a dict out
92 of the component nodes.
93 """
94 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)}
96 def visit_Set(self, node):
97 """This method is visited if the node is a set. It builds a set out
98 of the component nodes.
99 """
100 return {self.visit(el) for el in node.elts}
102 def visit_UnaryOp(self, node):
103 """This method is visited if the node is a unary operator. Currently
104 The only operator we support is the negative (-) operator, all others
105 are passed to generic_visit method.
106 """
107 if isinstance(node.op, ast.USub):
108 value = self.visit(node.operand)
109 return -1 * value
110 self.generic_visit(node)
112 def generic_visit(self, node):
113 """This method is called for all other node types. It will just raise
114 a value error, because this is a type of expression that we do not
115 support.
116 """
117 raise ValueError("Unable to parse string into literal expression")
120class ConfigOverrides:
121 """Defines a set of overrides to be applied to a task config.
123 Overrides for task configuration need to be applied by activator when
124 creating task instances. This class represents an ordered set of such
125 overrides which activator receives from some source (e.g. command line
126 or some other configuration).
128 Methods
129 ----------
130 addFileOverride(filename)
131 Add overrides from a specified file.
132 addValueOverride(field, value)
133 Add override for a specific field.
134 applyTo(config)
135 Apply all overrides to a `config` instance.
137 Notes
138 -----
139 Serialization support for this class may be needed, will add later if
140 necessary.
141 """
143 def __init__(self):
144 self._overrides = []
146 def addFileOverride(self, filename):
147 """Add overrides from a specified file.
149 Parameters
150 ----------
151 filename : convertible to `ResourcePath`
152 Path or URI to the override file. All URI schemes supported by
153 `ResourcePath` are supported.
154 """
155 self._overrides.append((OverrideTypes.File, ResourcePath(filename)))
157 def addValueOverride(self, field, value):
158 """Add override for a specific field.
160 This method is not very type-safe as it is designed to support
161 use cases where input is given as string, e.g. command line
162 activators. If `value` has a string type and setting of the field
163 fails with `TypeError` the we'll attempt `eval()` the value and
164 set the field with that value instead.
166 Parameters
167 ----------
168 field : str
169 Fully-qualified field name.
170 value :
171 Value to be given to a filed.
172 """
173 self._overrides.append((OverrideTypes.Value, (field, value)))
175 def addPythonOverride(self, python_snippet: str) -> None:
176 """Add Overrides by running a snippit of python code against a config.
178 Parameters
179 ----------
180 python_snippet: str
181 A string which is valid python code to be executed. This is done
182 with config as the only local accessible value.
183 """
184 self._overrides.append((OverrideTypes.Python, python_snippet))
186 def addInstrumentOverride(self, instrument: str, task_name: str) -> None:
187 """Apply any overrides that an instrument has for a task
189 Parameters
190 ----------
191 instrument: str
192 A string containing the fully qualified name of an instrument from
193 which configs should be loaded and applied
194 task_name: str
195 The _DefaultName of a task associated with a config, used to look
196 up overrides from the instrument.
197 """
198 instrument_cls: type = doImportType(instrument)
199 instrument_lib = instrument_cls()
200 self._overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name)))
202 def _parser(self, value, configParser):
203 try:
204 value = configParser.visit(ast.parse(value, mode="eval").body)
205 except Exception:
206 # This probably means it is a specific user string such as a URI.
207 # Let the value return as a string to attempt to continue to
208 # process as a string, another error will be raised in downstream
209 # code if that assumption is wrong
210 pass
212 return value
214 def applyTo(self, config):
215 """Apply all overrides to a task configuration object.
217 Parameters
218 ----------
219 config : `pex.Config`
220 Configuration to apply to; modified in place.
222 Raises
223 ------
224 `Exception` is raised if operations on configuration object fail.
225 """
226 # Look up a stack of variables people may be using when setting
227 # configs. Create a dictionary that will be used akin to a namespace
228 # for the duration of this function.
229 vars = {}
230 # pull in the variables that are declared in module scope of the config
231 mod = inspect.getmodule(config)
232 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")})
233 # put the supplied config in the variables dictionary
234 vars["config"] = config
236 # Create a parser for config expressions that may be strings
237 configParser = ConfigExpressionParser(namespace=vars)
239 for otype, override in self._overrides:
240 if otype is OverrideTypes.File:
241 with override.open("r") as buffer:
242 config.loadFromStream(buffer, filename=override.ospath)
243 elif otype is OverrideTypes.Value:
244 field, value = override
245 if isinstance(value, str):
246 value = self._parser(value, configParser)
247 # checking for dicts and lists here is needed because {} and []
248 # are valid yaml syntax so they get converted before hitting
249 # this method, so we must parse the elements.
250 #
251 # The same override would remain a string if specified on the
252 # command line, and is handled above.
253 if isinstance(value, dict):
254 new = {}
255 for k, v in value.items():
256 if isinstance(v, str):
257 new[k] = self._parser(v, configParser)
258 else:
259 new[k] = v
260 value = new
261 elif isinstance(value, list):
262 new = []
263 for v in value:
264 if isinstance(v, str):
265 new.append(self._parser(v, configParser))
266 else:
267 new.append(v)
268 value = new
269 # The field might be a string corresponding to a attribute
270 # hierarchy, attempt to split off the last field which
271 # will then be set.
272 parent, *child = field.rsplit(".", maxsplit=1)
273 if child:
274 # This branch means there was a hierarchy, get the
275 # field to set, and look up the sub config for which
276 # it is to be set
277 finalField = child[0]
278 tmpConfig = attrgetter(parent)(config)
279 else:
280 # There is no hierarchy, the config is the base config
281 # and the field is exactly what was passed in
282 finalField = parent
283 tmpConfig = config
284 # set the specified config
285 setattr(tmpConfig, finalField, value)
287 elif otype is OverrideTypes.Python:
288 # exec python string with the context of all vars known. This
289 # both lets people use a var they know about (maybe a bit
290 # dangerous, but not really more so than arbitrary python exec,
291 # and there are so many other places to worry about security
292 # before we start changing this) and any imports that are done
293 # in a python block will be put into this scope. This means
294 # other config setting branches can make use of these
295 # variables.
296 exec(override, None, vars)
297 elif otype is OverrideTypes.Instrument:
298 instrument, name = override
299 instrument.applyConfigOverrides(name, config)