Coverage for python/lsst/pipe/base/configOverrides.py: 23%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

90 statements  

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module which defines ConfigOverrides class and related methods. 

23""" 

24 

25__all__ = ["ConfigOverrides"] 

26 

27import ast 

28import inspect 

29from enum import Enum 

30from operator import attrgetter 

31 

32from lsst.utils import doImportType 

33 

34OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

35 

36 

37class ConfigExpressionParser(ast.NodeVisitor): 

38 """An expression parser that will be used to transform configuration 

39 strings supplied from the command line or a pipeline into a python 

40 object. 

41 

42 This is roughly equivalent to ast.literal_parser, but with the ability to 

43 transform strings that are valid variable names into the value associated 

44 with the name. Variables that should be considered valid are supplied to 

45 the constructor as a dictionary that maps a string to its corresponding 

46 value. 

47 

48 This class in an internal implementation detail, and should not be exposed 

49 outside this module. 

50 

51 Parameters 

52 ---------- 

53 namespace : `dict` of `str` to variable 

54 This is a mapping of strings corresponding to variable names, to the 

55 object that is associated with that name 

56 """ 

57 

58 def __init__(self, namespace): 

59 self.variables = namespace 

60 

61 def visit_Name(self, node): 

62 """This method gets called when the parser has determined a node 

63 corresponds to a variable name. 

64 """ 

65 # If the id (name) of the variable is in the dictionary of valid names, 

66 # load and return the corresponding variable. 

67 if node.id in self.variables: 

68 return self.variables[node.id] 

69 # If the node does not correspond to a valid variable, turn the name 

70 # into a string, as the user likely intended it as such. 

71 return f"{node.id}" 

72 

73 def visit_List(self, node): 

74 """This method is visited if the node is a list. Constructs a list out 

75 of the sub nodes. 

76 """ 

77 return [self.visit(elm) for elm in node.elts] 

78 

79 def visit_Tuple(self, node): 

80 """This method is visited if the node is a tuple. Constructs a list out 

81 of the sub nodes, and then turns it into a tuple. 

82 """ 

83 return tuple(self.visit_List(node)) 

84 

85 def visit_Constant(self, node): 

86 """This method is visited if the node is a constant""" 

87 return node.value 

88 

89 def visit_Dict(self, node): 

90 """This method is visited if the node is a dict. It builds a dict out 

91 of the component nodes. 

92 """ 

93 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)} 

94 

95 def visit_Set(self, node): 

96 """This method is visited if the node is a set. It builds a set out 

97 of the component nodes. 

98 """ 

99 return {self.visit(el) for el in node.elts} 

100 

101 def visit_UnaryOp(self, node): 

102 """This method is visited if the node is a unary operator. Currently 

103 The only operator we support is the negative (-) operator, all others 

104 are passed to generic_visit method. 

105 """ 

106 if isinstance(node.op, ast.USub): 

107 value = self.visit(node.operand) 

108 return -1 * value 

109 self.generic_visit(node) 

110 

111 def generic_visit(self, node): 

112 """This method is called for all other node types. It will just raise 

113 a value error, because this is a type of expression that we do not 

114 support. 

115 """ 

116 raise ValueError("Unable to parse string into literal expression") 

117 

118 

119class ConfigOverrides: 

120 """Defines a set of overrides to be applied to a task config. 

121 

122 Overrides for task configuration need to be applied by activator when 

123 creating task instances. This class represents an ordered set of such 

124 overrides which activator receives from some source (e.g. command line 

125 or some other configuration). 

126 

127 Methods 

128 ---------- 

129 addFileOverride(filename) 

130 Add overrides from a specified file. 

131 addValueOverride(field, value) 

132 Add override for a specific field. 

133 applyTo(config) 

134 Apply all overrides to a `config` instance. 

135 

136 Notes 

137 ----- 

138 Serialization support for this class may be needed, will add later if 

139 necessary. 

140 """ 

141 

142 def __init__(self): 

143 self._overrides = [] 

144 

145 def addFileOverride(self, filename): 

146 """Add overrides from a specified file. 

147 

148 Parameters 

149 ---------- 

150 filename : str 

151 Path to the override file. 

152 """ 

153 self._overrides.append((OverrideTypes.File, filename)) 

154 

155 def addValueOverride(self, field, value): 

156 """Add override for a specific field. 

157 

158 This method is not very type-safe as it is designed to support 

159 use cases where input is given as string, e.g. command line 

160 activators. If `value` has a string type and setting of the field 

161 fails with `TypeError` the we'll attempt `eval()` the value and 

162 set the field with that value instead. 

163 

164 Parameters 

165 ---------- 

166 field : str 

167 Fully-qualified field name. 

168 value : 

169 Value to be given to a filed. 

170 """ 

171 self._overrides.append((OverrideTypes.Value, (field, value))) 

172 

173 def addPythonOverride(self, python_snippet: str) -> None: 

174 """Add Overrides by running a snippit of python code against a config. 

175 

176 Parameters 

177 ---------- 

178 python_snippet: str 

179 A string which is valid python code to be executed. This is done 

180 with config as the only local accessible value. 

181 """ 

182 self._overrides.append((OverrideTypes.Python, python_snippet)) 

183 

184 def addInstrumentOverride(self, instrument: str, task_name: str) -> None: 

185 """Apply any overrides that an instrument has for a task 

186 

187 Parameters 

188 ---------- 

189 instrument: str 

190 A string containing the fully qualified name of an instrument from 

191 which configs should be loaded and applied 

192 task_name: str 

193 The _DefaultName of a task associated with a config, used to look 

194 up overrides from the instrument. 

195 """ 

196 instrument_cls: type = doImportType(instrument) 

197 instrument_lib = instrument_cls() 

198 self._overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name))) 

199 

200 def _parser(self, value, configParser): 

201 try: 

202 value = configParser.visit(ast.parse(value, mode="eval").body) 

203 except Exception: 

204 # This probably means it is a specific user string such as a URI. 

205 # Let the value return as a string to attempt to continue to 

206 # process as a string, another error will be raised in downstream 

207 # code if that assumption is wrong 

208 pass 

209 

210 return value 

211 

212 def applyTo(self, config): 

213 """Apply all overrides to a task configuration object. 

214 

215 Parameters 

216 ---------- 

217 config : `pex.Config` 

218 

219 Raises 

220 ------ 

221 `Exception` is raised if operations on configuration object fail. 

222 """ 

223 # Look up a stack of variables people may be using when setting 

224 # configs. Create a dictionary that will be used akin to a namespace 

225 # for the duration of this function. 

226 vars = {} 

227 # pull in the variables that are declared in module scope of the config 

228 mod = inspect.getmodule(config) 

229 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

230 # put the supplied config in the variables dictionary 

231 vars["config"] = config 

232 

233 # Create a parser for config expressions that may be strings 

234 configParser = ConfigExpressionParser(namespace=vars) 

235 

236 for otype, override in self._overrides: 

237 if otype is OverrideTypes.File: 

238 config.load(override) 

239 elif otype is OverrideTypes.Value: 

240 field, value = override 

241 if isinstance(value, str): 

242 value = self._parser(value, configParser) 

243 # checking for dicts and lists here is needed because {} and [] 

244 # are valid yaml syntax so they get converted before hitting 

245 # this method, so we must parse the elements. 

246 # 

247 # The same override would remain a string if specified on the 

248 # command line, and is handled above. 

249 if isinstance(value, dict): 

250 new = {} 

251 for k, v in value.items(): 

252 if isinstance(v, str): 

253 new[k] = self._parser(v, configParser) 

254 else: 

255 new[k] = v 

256 value = new 

257 elif isinstance(value, list): 

258 new = [] 

259 for v in value: 

260 if isinstance(v, str): 

261 new.append(self._parser(v, configParser)) 

262 else: 

263 new.append(v) 

264 value = new 

265 # The field might be a string corresponding to a attribute 

266 # hierarchy, attempt to split off the last field which 

267 # will then be set. 

268 parent, *child = field.rsplit(".", maxsplit=1) 

269 if child: 

270 # This branch means there was a hierarchy, get the 

271 # field to set, and look up the sub config for which 

272 # it is to be set 

273 finalField = child[0] 

274 tmpConfig = attrgetter(parent)(config) 

275 else: 

276 # There is no hierarchy, the config is the base config 

277 # and the field is exactly what was passed in 

278 finalField = parent 

279 tmpConfig = config 

280 # set the specified config 

281 setattr(tmpConfig, finalField, value) 

282 

283 elif otype is OverrideTypes.Python: 

284 # exec python string with the context of all vars known. This 

285 # both lets people use a var they know about (maybe a bit 

286 # dangerous, but not really more so than arbitrary python exec, 

287 # and there are so many other places to worry about security 

288 # before we start changing this) and any imports that are done 

289 # in a python block will be put into this scope. This means 

290 # other config setting branches can make use of these 

291 # variables. 

292 exec(override, None, vars) 

293 elif otype is OverrideTypes.Instrument: 

294 instrument, name = override 

295 instrument.applyConfigOverrides(name, config)