Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module which defines ConfigOverrides class and related methods. 

23""" 

24 

25__all__ = ["ConfigOverrides"] 

26 

27import ast 

28from operator import attrgetter 

29 

30from lsst.utils import doImport 

31 

32import inspect 

33from enum import Enum 

34 

35OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

36 

37 

38class ConfigExpressionParser(ast.NodeVisitor): 

39 """An expression parser that will be used to transform configuration 

40 strings supplied from the command line or a pipeline into a python 

41 object. 

42 

43 This is roughly equivalent to ast.literal_parser, but with the ability to 

44 transform strings that are valid variable names into the value associated 

45 with the name. Variables that should be considered valid are supplied to 

46 the constructor as a dictionary that maps a string to its corresponding 

47 value. 

48 

49 This class in an internal implementation detail, and should not be exposed 

50 outside this module. 

51 

52 Parameters 

53 ---------- 

54 namespace : `dict` of `str` to variable 

55 This is a mapping of strings corresponding to variable names, to the 

56 object that is associated with that name 

57 """ 

58 

59 def __init__(self, namespace): 

60 self.variables = namespace 

61 

62 def visit_Name(self, node): 

63 """This method gets called when the parser has determined a node 

64 corresponds to a variable name. 

65 """ 

66 # If the id (name) of the variable is in the dictionary of valid names, 

67 # load and return the corresponding variable. 

68 if node.id in self.variables: 

69 return self.variables[node.id] 

70 # If the node does not correspond to a valid variable, turn the name 

71 # into a string, as the user likely intended it as such. 

72 return f"{node.id}" 

73 

74 def visit_List(self, node): 

75 """This method is visited if the node is a list. Constructs a list out 

76 of the sub nodes. 

77 """ 

78 return [self.visit(elm) for elm in node.elts] 

79 

80 def visit_Tuple(self, node): 

81 """This method is visited if the node is a tuple. Constructs a list out 

82 of the sub nodes, and then turns it into a tuple. 

83 """ 

84 return tuple(self.visit_List(node)) 

85 

86 def visit_Constant(self, node): 

87 """This method is visited if the node is a constant 

88 """ 

89 return node.value 

90 

91 def visit_Dict(self, node): 

92 """This method is visited if the node is a dict. It builds a dict out 

93 of the component nodes. 

94 """ 

95 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)} 

96 

97 def visit_Set(self, node): 

98 """This method is visited if the node is a set. It builds a set out 

99 of the component nodes. 

100 """ 

101 return {self.visit(el) for el in node.elts} 

102 

103 def visit_UnaryOp(self, node): 

104 """This method is visited if the node is a unary operator. Currently 

105 The only operator we support is the negative (-) operator, all others 

106 are passed to generic_visit method. 

107 """ 

108 if isinstance(node.op, ast.USub): 

109 value = self.visit(node.operand) 

110 return -1*value 

111 self.generic_visit(node) 

112 

113 def generic_visit(self, node): 

114 """This method is called for all other node types. It will just raise 

115 a value error, because this is a type of expression that we do not 

116 support. 

117 """ 

118 raise ValueError("Unable to parse string into literal expression") 

119 

120 

121class ConfigOverrides: 

122 """Defines a set of overrides to be applied to a task config. 

123 

124 Overrides for task configuration need to be applied by activator when 

125 creating task instances. This class represents an ordered set of such 

126 overrides which activator receives from some source (e.g. command line 

127 or some other configuration). 

128 

129 Methods 

130 ---------- 

131 addFileOverride(filename) 

132 Add overrides from a specified file. 

133 addValueOverride(field, value) 

134 Add override for a specific field. 

135 applyTo(config) 

136 Apply all overrides to a `config` instance. 

137 

138 Notes 

139 ----- 

140 Serialization support for this class may be needed, will add later if 

141 necessary. 

142 """ 

143 

144 def __init__(self): 

145 self._overrides = [] 

146 

147 def addFileOverride(self, filename): 

148 """Add overrides from a specified file. 

149 

150 Parameters 

151 ---------- 

152 filename : str 

153 Path to the override file. 

154 """ 

155 self._overrides.append((OverrideTypes.File, filename)) 

156 

157 def addValueOverride(self, field, value): 

158 """Add override for a specific field. 

159 

160 This method is not very type-safe as it is designed to support 

161 use cases where input is given as string, e.g. command line 

162 activators. If `value` has a string type and setting of the field 

163 fails with `TypeError` the we'll attempt `eval()` the value and 

164 set the field with that value instead. 

165 

166 Parameters 

167 ---------- 

168 field : str 

169 Fully-qualified field name. 

170 value : 

171 Value to be given to a filed. 

172 """ 

173 self._overrides.append((OverrideTypes.Value, (field, value))) 

174 

175 def addPythonOverride(self, python_snippet: str): 

176 """Add Overrides by running a snippit of python code against a config. 

177 

178 Parameters 

179 ---------- 

180 python_snippet: str 

181 A string which is valid python code to be executed. This is done 

182 with config as the only local accessible value. 

183 """ 

184 self._overrides.append((OverrideTypes.Python, python_snippet)) 

185 

186 def addInstrumentOverride(self, instrument: str, task_name: str): 

187 """Apply any overrides that an instrument has for a task 

188 

189 Parameters 

190 ---------- 

191 instrument: str 

192 A string containing the fully qualified name of an instrument from 

193 which configs should be loaded and applied 

194 task_name: str 

195 The _DefaultName of a task associated with a config, used to look 

196 up overrides from the instrument. 

197 """ 

198 instrument_lib = doImport(instrument)() 

199 self._overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name))) 

200 

201 def _parser(self, value, configParser): 

202 try: 

203 value = configParser.visit(ast.parse(value, mode='eval').body) 

204 except Exception: 

205 # This probably means it is a specific user string such as a URI. 

206 # Let the value return as a string to attempt to continue to 

207 # process as a string, another error will be raised in downstream 

208 # code if that assumption is wrong 

209 pass 

210 

211 return value 

212 

213 def applyTo(self, config): 

214 """Apply all overrides to a task configuration object. 

215 

216 Parameters 

217 ---------- 

218 config : `pex.Config` 

219 

220 Raises 

221 ------ 

222 `Exception` is raised if operations on configuration object fail. 

223 """ 

224 # Look up a stack of variables people may be using when setting 

225 # configs. Create a dictionary that will be used akin to a namespace 

226 # for the duration of this function. 

227 vars = {} 

228 # pull in the variables that are declared in module scope of the config 

229 mod = inspect.getmodule(config) 

230 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

231 # put the supplied config in the variables dictionary 

232 vars['config'] = config 

233 

234 # Create a parser for config expressions that may be strings 

235 configParser = ConfigExpressionParser(namespace=vars) 

236 

237 for otype, override in self._overrides: 

238 if otype is OverrideTypes.File: 

239 config.load(override) 

240 elif otype is OverrideTypes.Value: 

241 field, value = override 

242 if isinstance(value, str): 

243 value = self._parser(value, configParser) 

244 # checking for dicts and lists here is needed because {} and [] 

245 # are valid yaml syntax so they get converted before hitting 

246 # this method, so we must parse the elements. 

247 # 

248 # The same override would remain a string if specified on the 

249 # command line, and is handled above. 

250 if isinstance(value, dict): 

251 new = {} 

252 for k, v in value.items(): 

253 if isinstance(v, str): 

254 new[k] = self._parser(v, configParser) 

255 else: 

256 new[k] = v 

257 value = new 

258 elif isinstance(value, list): 

259 new = [] 

260 for v in value: 

261 if isinstance(v, str): 

262 new.append(self._parser(v, configParser)) 

263 else: 

264 new.append(v) 

265 value = new 

266 # The field might be a string corresponding to a attribute 

267 # hierarchy, attempt to split off the last field which 

268 # will then be set. 

269 parent, *child = field.rsplit(".", maxsplit=1) 

270 if child: 

271 # This branch means there was a hierarchy, get the 

272 # field to set, and look up the sub config for which 

273 # it is to be set 

274 finalField = child[0] 

275 tmpConfig = attrgetter(parent)(config) 

276 else: 

277 # There is no hierarchy, the config is the base config 

278 # and the field is exactly what was passed in 

279 finalField = parent 

280 tmpConfig = config 

281 # set the specified config 

282 setattr(tmpConfig, finalField, value) 

283 

284 elif otype is OverrideTypes.Python: 

285 # exec python string with the context of all vars known. This 

286 # both lets people use a var they know about (maybe a bit 

287 # dangerous, but not really more so than arbitrary python exec, 

288 # and there are so many other places to worry about security 

289 # before we start changing this) and any imports that are done 

290 # in a python block will be put into this scope. This means 

291 # other config setting branches can make use of these 

292 # variables. 

293 exec(override, None, vars) 

294 elif otype is OverrideTypes.Instrument: 

295 instrument, name = override 

296 instrument.applyConfigOverrides(name, config)