Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module which defines ConfigOverrides class and related methods. 

23""" 

24 

25__all__ = ["ConfigOverrides"] 

26 

27import ast 

28from operator import attrgetter 

29 

30import lsst.pex.exceptions as pexExceptions 

31from lsst.utils import doImport 

32 

33import inspect 

34from enum import Enum 

35 

36OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

37 

38 

39class ConfigExpressionParser(ast.NodeVisitor): 

40 """An expression parser that will be used to transform configuration 

41 strings supplied from the command line or a pipeline into a python 

42 object. 

43 

44 This is roughly equivalent to ast.literal_parser, but with the ability to 

45 transform strings that are valid variable names into the value associated 

46 with the name. Variables that should be considered valid are supplied to 

47 the constructor as a dictionary that maps a string to its corresponding 

48 value. 

49 

50 This class in an internal implementation detail, and should not be exposed 

51 outside this module. 

52 

53 Parameters 

54 ---------- 

55 namespace : `dict` of `str` to variable 

56 This is a mapping of strings corresponding to variable names, to the 

57 object that is associated with that name 

58 """ 

59 

60 def __init__(self, namespace): 

61 self.variables = namespace 

62 

63 def visit_Name(self, node): 

64 """This method gets called when the parser has determined a node 

65 corresponds to a variable name. 

66 """ 

67 # If the id (name) of the variable is in the dictionary of valid names, 

68 # load and return the corresponding variable. 

69 if node.id in self.variables: 

70 return self.variables[node.id] 

71 # If the node does not correspond to a valid variable, turn the name 

72 # into a string, as the user likely intended it as such. 

73 return f"{node.id}" 

74 

75 def visit_List(self, node): 

76 """This method is visited if the node is a list. Constructs a list out 

77 of the sub nodes. 

78 """ 

79 return [self.visit(elm) for elm in node.elts] 

80 

81 def visit_Tuple(self, node): 

82 """This method is visited if the node is a tuple. Constructs a list out 

83 of the sub nodes, and then turns it into a tuple. 

84 """ 

85 return tuple(self.visit_List(node)) 

86 

87 def visit_Constant(self, node): 

88 """This method is visited if the node is a constant 

89 """ 

90 return node.value 

91 

92 def visit_Dict(self, node): 

93 """This method is visited if the node is a dict. It builds a dict out 

94 of the component nodes. 

95 """ 

96 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)} 

97 

98 def visit_Set(self, node): 

99 """This method is visited if the node is a set. It builds a set out 

100 of the component nodes. 

101 """ 

102 return {self.visit(el) for el in node.elts} 

103 

104 def visit_UnaryOp(self, node): 

105 """This method is visited if the node is a unary operator. Currently 

106 The only operator we support is the negative (-) operator, all others 

107 are passed to generic_visit method. 

108 """ 

109 if isinstance(node.op, ast.USub): 

110 value = self.visit(node.operand) 

111 return -1*value 

112 self.generic_visit(node) 

113 

114 def generic_visit(self, node): 

115 """This method is called for all other node types. It will just raise 

116 a value error, because this is a type of expression that we do not 

117 support. 

118 """ 

119 raise ValueError("Unable to parse string into literal expression") 

120 

121 

122class ConfigOverrides: 

123 """Defines a set of overrides to be applied to a task config. 

124 

125 Overrides for task configuration need to be applied by activator when 

126 creating task instances. This class represents an ordered set of such 

127 overrides which activator receives from some source (e.g. command line 

128 or some other configuration). 

129 

130 Methods 

131 ---------- 

132 addFileOverride(filename) 

133 Add overrides from a specified file. 

134 addValueOverride(field, value) 

135 Add override for a specific field. 

136 applyTo(config) 

137 Apply all overrides to a `config` instance. 

138 

139 Notes 

140 ----- 

141 Serialization support for this class may be needed, will add later if 

142 necessary. 

143 """ 

144 

145 def __init__(self): 

146 self._overrides = [] 

147 

148 def addFileOverride(self, filename): 

149 """Add overrides from a specified file. 

150 

151 Parameters 

152 ---------- 

153 filename : str 

154 Path to the override file. 

155 """ 

156 self._overrides.append((OverrideTypes.File, filename)) 

157 

158 def addValueOverride(self, field, value): 

159 """Add override for a specific field. 

160 

161 This method is not very type-safe as it is designed to support 

162 use cases where input is given as string, e.g. command line 

163 activators. If `value` has a string type and setting of the field 

164 fails with `TypeError` the we'll attempt `eval()` the value and 

165 set the field with that value instead. 

166 

167 Parameters 

168 ---------- 

169 field : str 

170 Fully-qualified field name. 

171 value : 

172 Value to be given to a filed. 

173 """ 

174 self._overrides.append((OverrideTypes.Value, (field, value))) 

175 

176 def addPythonOverride(self, python_snippet: str): 

177 """Add Overrides by running a snippit of python code against a config. 

178 

179 Parameters 

180 ---------- 

181 python_snippet: str 

182 A string which is valid python code to be executed. This is done 

183 with config as the only local accessible value. 

184 """ 

185 self._overrides.append((OverrideTypes.Python, python_snippet)) 

186 

187 def addInstrumentOverride(self, instrument: str, task_name: str): 

188 """Apply any overrides that an instrument has for a task 

189 

190 Parameters 

191 ---------- 

192 instrument: str 

193 A string containing the fully qualified name of an instrument from 

194 which configs should be loaded and applied 

195 task_name: str 

196 The _DefaultName of a task associated with a config, used to look 

197 up overrides from the instrument. 

198 """ 

199 instrument_lib = doImport(instrument)() 

200 self._overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name))) 

201 

202 def _parser(self, value, configParser): 

203 try: 

204 value = configParser.visit(ast.parse(value, mode='eval').body) 

205 except ValueError: 

206 # eval failed, wrap exception with more user-friendly 

207 # message 

208 raise pexExceptions.RuntimeError(f"Unable to parse `{value}' into a Python object") from None 

209 return value 

210 

211 def applyTo(self, config): 

212 """Apply all overrides to a task configuration object. 

213 

214 Parameters 

215 ---------- 

216 config : `pex.Config` 

217 

218 Raises 

219 ------ 

220 `Exception` is raised if operations on configuration object fail. 

221 """ 

222 # Look up a stack of variables people may be using when setting 

223 # configs. Create a dictionary that will be used akin to a namespace 

224 # for the duration of this function. 

225 vars = {} 

226 # pull in the variables that are declared in module scope of the config 

227 mod = inspect.getmodule(config) 

228 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

229 # put the supplied config in the variables dictionary 

230 vars['config'] = config 

231 

232 # Create a parser for config expressions that may be strings 

233 configParser = ConfigExpressionParser(namespace=vars) 

234 

235 for otype, override in self._overrides: 

236 if otype is OverrideTypes.File: 

237 config.load(override) 

238 elif otype is OverrideTypes.Value: 

239 field, value = override 

240 if isinstance(value, str): 

241 value = self._parser(value, configParser) 

242 # checking for dicts and lists here is needed because {} and [] 

243 # are valid yaml syntax so they get converted before hitting 

244 # this method, so we must parse the elements. 

245 # 

246 # The same override would remain a string if specified on the 

247 # command line, and is handled above. 

248 if isinstance(value, dict): 

249 new = {} 

250 for k, v in value.items(): 

251 if isinstance(v, str): 

252 new[k] = self._parser(v, configParser) 

253 else: 

254 new[k] = v 

255 value = new 

256 elif isinstance(value, list): 

257 new = [] 

258 for v in value: 

259 if isinstance(v, str): 

260 new.append(self._parser(v, configParser)) 

261 else: 

262 new.append(v) 

263 value = new 

264 # The field might be a string corresponding to a attribute 

265 # hierarchy, attempt to split off the last field which 

266 # will then be set. 

267 parent, *child = field.rsplit(".", maxsplit=1) 

268 if child: 

269 # This branch means there was a hierarchy, get the 

270 # field to set, and look up the sub config for which 

271 # it is to be set 

272 finalField = child[0] 

273 tmpConfig = attrgetter(parent)(config) 

274 else: 

275 # There is no hierarchy, the config is the base config 

276 # and the field is exactly what was passed in 

277 finalField = parent 

278 tmpConfig = config 

279 # set the specified config 

280 setattr(tmpConfig, finalField, value) 

281 

282 elif otype is OverrideTypes.Python: 

283 # exec python string with the context of all vars known. This 

284 # both lets people use a var they know about (maybe a bit 

285 # dangerous, but not really more so than arbitrary python exec, 

286 # and there are so many other places to worry about security 

287 # before we start changing this) and any imports that are done 

288 # in a python block will be put into this scope. This means 

289 # other config setting branches can make use of these 

290 # variables. 

291 exec(override, None, vars) 

292 elif otype is OverrideTypes.Instrument: 

293 instrument, name = override 

294 instrument.applyConfigOverrides(name, config)