lsst.pipe.base  21.0.0-13-g03fae8e+dd7c5f6b68
configOverrides.py
Go to the documentation of this file.
1 # This file is part of pipe_base.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (http://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <http://www.gnu.org/licenses/>.
21 
22 """Module which defines ConfigOverrides class and related methods.
23 """
24 
25 __all__ = ["ConfigOverrides"]
26 
27 import ast
28 from operator import attrgetter
29 
30 import lsst.pex.exceptions as pexExceptions
31 from lsst.utils import doImport
32 
33 import inspect
34 from enum import Enum
35 
36 OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument")
37 
38 
39 class ConfigExpressionParser(ast.NodeVisitor):
40  """An expression parser that will be used to transform configuration
41  strings supplied from the command line or a pipeline into a python
42  object.
43 
44  This is roughly equivalent to ast.literal_parser, but with the ability to
45  transform strings that are valid variable names into the value associated
46  with the name. Variables that should be considered valid are supplied to
47  the constructor as a dictionary that maps a string to its corresponding
48  value.
49 
50  This class in an internal implementation detail, and should not be exposed
51  outside this module.
52 
53  Parameters
54  ----------
55  namespace : `dict` of `str` to variable
56  This is a mapping of strings corresponding to variable names, to the
57  object that is associated with that name
58  """
59 
60  def __init__(self, namespace):
61  self.variablesvariables = namespace
62 
63  def visit_Name(self, node):
64  """This method gets called when the parser has determined a node
65  corresponds to a variable name.
66  """
67  # If the id (name) of the variable is in the dictionary of valid names,
68  # load and return the corresponding variable.
69  if node.id in self.variablesvariables:
70  return self.variablesvariables[node.id]
71  # If the node does not correspond to a valid variable, turn the name
72  # into a string, as the user likely intended it as such.
73  return f"{node.id}"
74 
75  def visit_List(self, node):
76  """This method is visited if the node is a list. Constructs a list out
77  of the sub nodes.
78  """
79  return [self.visit(elm) for elm in node.elts]
80 
81  def visit_Tuple(self, node):
82  """This method is visited if the node is a tuple. Constructs a list out
83  of the sub nodes, and then turns it into a tuple.
84  """
85  return tuple(self.visit_Listvisit_List(node))
86 
87  def visit_Constant(self, node):
88  """This method is visited if the node is a constant
89  """
90  return node.value
91 
92  def visit_Dict(self, node):
93  """This method is visited if the node is a dict. It builds a dict out
94  of the component nodes.
95  """
96  return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)}
97 
98  def visit_Set(self, node):
99  """This method is visited if the node is a set. It builds a set out
100  of the component nodes.
101  """
102  return {self.visit(el) for el in node.elts}
103 
104  def visit_UnaryOp(self, node):
105  """This method is visited if the node is a unary operator. Currently
106  The only operator we support is the negative (-) operator, all others
107  are passed to generic_visit method.
108  """
109  if isinstance(node.op, ast.USub):
110  value = self.visit(node.operand)
111  return -1*value
112  self.generic_visitgeneric_visit(node)
113 
114  def generic_visit(self, node):
115  """This method is called for all other node types. It will just raise
116  a value error, because this is a type of expression that we do not
117  support.
118  """
119  raise ValueError("Unable to parse string into literal expression")
120 
121 
123  """Defines a set of overrides to be applied to a task config.
124 
125  Overrides for task configuration need to be applied by activator when
126  creating task instances. This class represents an ordered set of such
127  overrides which activator receives from some source (e.g. command line
128  or some other configuration).
129 
130  Methods
131  ----------
132  addFileOverride(filename)
133  Add overrides from a specified file.
134  addValueOverride(field, value)
135  Add override for a specific field.
136  applyTo(config)
137  Apply all overrides to a `config` instance.
138 
139  Notes
140  -----
141  Serialization support for this class may be needed, will add later if
142  necessary.
143  """
144 
145  def __init__(self):
146  self._overrides_overrides = []
147 
148  def addFileOverride(self, filename):
149  """Add overrides from a specified file.
150 
151  Parameters
152  ----------
153  filename : str
154  Path to the override file.
155  """
156  self._overrides_overrides.append((OverrideTypes.File, filename))
157 
158  def addValueOverride(self, field, value):
159  """Add override for a specific field.
160 
161  This method is not very type-safe as it is designed to support
162  use cases where input is given as string, e.g. command line
163  activators. If `value` has a string type and setting of the field
164  fails with `TypeError` the we'll attempt `eval()` the value and
165  set the field with that value instead.
166 
167  Parameters
168  ----------
169  field : str
170  Fully-qualified field name.
171  value :
172  Value to be given to a filed.
173  """
174  self._overrides_overrides.append((OverrideTypes.Value, (field, value)))
175 
176  def addPythonOverride(self, python_snippet: str):
177  """Add Overrides by running a snippit of python code against a config.
178 
179  Parameters
180  ----------
181  python_snippet: str
182  A string which is valid python code to be executed. This is done
183  with config as the only local accessible value.
184  """
185  self._overrides_overrides.append((OverrideTypes.Python, python_snippet))
186 
187  def addInstrumentOverride(self, instrument: str, task_name: str):
188  """Apply any overrides that an instrument has for a task
189 
190  Parameters
191  ----------
192  instrument: str
193  A string containing the fully qualified name of an instrument from
194  which configs should be loaded and applied
195  task_name: str
196  The _DefaultName of a task associated with a config, used to look
197  up overrides from the instrument.
198  """
199  instrument_lib = doImport(instrument)()
200  self._overrides_overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name)))
201 
202  def _parser(self, value, configParser):
203  try:
204  value = configParser.visit(ast.parse(value, mode='eval').body)
205  except ValueError:
206  # eval failed, wrap exception with more user-friendly
207  # message
208  raise pexExceptions.RuntimeError(f"Unable to parse `{value}' into a Python object") from None
209  return value
210 
211  def applyTo(self, config):
212  """Apply all overrides to a task configuration object.
213 
214  Parameters
215  ----------
216  config : `pex.Config`
217 
218  Raises
219  ------
220  `Exception` is raised if operations on configuration object fail.
221  """
222  # Look up a stack of variables people may be using when setting
223  # configs. Create a dictionary that will be used akin to a namespace
224  # for the duration of this function.
225  vars = {}
226  # pull in the variables that are declared in module scope of the config
227  mod = inspect.getmodule(config)
228  vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")})
229  # put the supplied config in the variables dictionary
230  vars['config'] = config
231 
232  # Create a parser for config expressions that may be strings
233  configParser = ConfigExpressionParser(namespace=vars)
234 
235  for otype, override in self._overrides_overrides:
236  if otype is OverrideTypes.File:
237  config.load(override)
238  elif otype is OverrideTypes.Value:
239  field, value = override
240  if isinstance(value, str):
241  value = self._parser_parser(value, configParser)
242  # checking for dicts and lists here is needed because {} and []
243  # are valid yaml syntax so they get converted before hitting
244  # this method, so we must parse the elements.
245  #
246  # The same override would remain a string if specified on the
247  # command line, and is handled above.
248  if isinstance(value, dict):
249  new = {}
250  for k, v in value.items():
251  if isinstance(v, str):
252  new[k] = self._parser_parser(v, configParser)
253  else:
254  new[k] = v
255  value = new
256  elif isinstance(value, list):
257  new = []
258  for v in value:
259  if isinstance(v, str):
260  new.append(self._parser_parser(v, configParser))
261  else:
262  new.append(v)
263  value = new
264  # The field might be a string corresponding to a attribute
265  # hierarchy, attempt to split off the last field which
266  # will then be set.
267  parent, *child = field.rsplit(".", maxsplit=1)
268  if child:
269  # This branch means there was a hierarchy, get the
270  # field to set, and look up the sub config for which
271  # it is to be set
272  finalField = child[0]
273  tmpConfig = attrgetter(parent)(config)
274  else:
275  # There is no hierarchy, the config is the base config
276  # and the field is exactly what was passed in
277  finalField = parent
278  tmpConfig = config
279  # set the specified config
280  setattr(tmpConfig, finalField, value)
281 
282  elif otype is OverrideTypes.Python:
283  # exec python string with the context of all vars known. This
284  # both lets people use a var they know about (maybe a bit
285  # dangerous, but not really more so than arbitrary python exec,
286  # and there are so many other places to worry about security
287  # before we start changing this) and any imports that are done
288  # in a python block will be put into this scope. This means
289  # other config setting branches can make use of these
290  # variables.
291  exec(override, None, vars)
292  elif otype is OverrideTypes.Instrument:
293  instrument, name = override
294  instrument.applyConfigOverrides(name, config)
def addPythonOverride(self, str python_snippet)
def addInstrumentOverride(self, str instrument, str task_name)