Coverage for python/lsst/pipe/base/configOverrides.py: 23%

92 statements  

« prev     ^ index     » next       coverage.py v6.4.4, created at 2022-08-20 08:52 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module which defines ConfigOverrides class and related methods. 

23""" 

24 

25__all__ = ["ConfigOverrides"] 

26 

27import ast 

28import inspect 

29from enum import Enum 

30from operator import attrgetter 

31 

32from lsst.resources import ResourcePath 

33from lsst.utils import doImportType 

34 

35OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

36 

37 

38class ConfigExpressionParser(ast.NodeVisitor): 

39 """An expression parser that will be used to transform configuration 

40 strings supplied from the command line or a pipeline into a python 

41 object. 

42 

43 This is roughly equivalent to ast.literal_parser, but with the ability to 

44 transform strings that are valid variable names into the value associated 

45 with the name. Variables that should be considered valid are supplied to 

46 the constructor as a dictionary that maps a string to its corresponding 

47 value. 

48 

49 This class in an internal implementation detail, and should not be exposed 

50 outside this module. 

51 

52 Parameters 

53 ---------- 

54 namespace : `dict` of `str` to variable 

55 This is a mapping of strings corresponding to variable names, to the 

56 object that is associated with that name 

57 """ 

58 

59 def __init__(self, namespace): 

60 self.variables = namespace 

61 

62 def visit_Name(self, node): 

63 """This method gets called when the parser has determined a node 

64 corresponds to a variable name. 

65 """ 

66 # If the id (name) of the variable is in the dictionary of valid names, 

67 # load and return the corresponding variable. 

68 if node.id in self.variables: 

69 return self.variables[node.id] 

70 # If the node does not correspond to a valid variable, turn the name 

71 # into a string, as the user likely intended it as such. 

72 return f"{node.id}" 

73 

74 def visit_List(self, node): 

75 """This method is visited if the node is a list. Constructs a list out 

76 of the sub nodes. 

77 """ 

78 return [self.visit(elm) for elm in node.elts] 

79 

80 def visit_Tuple(self, node): 

81 """This method is visited if the node is a tuple. Constructs a list out 

82 of the sub nodes, and then turns it into a tuple. 

83 """ 

84 return tuple(self.visit_List(node)) 

85 

86 def visit_Constant(self, node): 

87 """This method is visited if the node is a constant""" 

88 return node.value 

89 

90 def visit_Dict(self, node): 

91 """This method is visited if the node is a dict. It builds a dict out 

92 of the component nodes. 

93 """ 

94 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values)} 

95 

96 def visit_Set(self, node): 

97 """This method is visited if the node is a set. It builds a set out 

98 of the component nodes. 

99 """ 

100 return {self.visit(el) for el in node.elts} 

101 

102 def visit_UnaryOp(self, node): 

103 """This method is visited if the node is a unary operator. Currently 

104 The only operator we support is the negative (-) operator, all others 

105 are passed to generic_visit method. 

106 """ 

107 if isinstance(node.op, ast.USub): 

108 value = self.visit(node.operand) 

109 return -1 * value 

110 self.generic_visit(node) 

111 

112 def generic_visit(self, node): 

113 """This method is called for all other node types. It will just raise 

114 a value error, because this is a type of expression that we do not 

115 support. 

116 """ 

117 raise ValueError("Unable to parse string into literal expression") 

118 

119 

120class ConfigOverrides: 

121 """Defines a set of overrides to be applied to a task config. 

122 

123 Overrides for task configuration need to be applied by activator when 

124 creating task instances. This class represents an ordered set of such 

125 overrides which activator receives from some source (e.g. command line 

126 or some other configuration). 

127 

128 Methods 

129 ---------- 

130 addFileOverride(filename) 

131 Add overrides from a specified file. 

132 addValueOverride(field, value) 

133 Add override for a specific field. 

134 applyTo(config) 

135 Apply all overrides to a `config` instance. 

136 

137 Notes 

138 ----- 

139 Serialization support for this class may be needed, will add later if 

140 necessary. 

141 """ 

142 

143 def __init__(self): 

144 self._overrides = [] 

145 

146 def addFileOverride(self, filename): 

147 """Add overrides from a specified file. 

148 

149 Parameters 

150 ---------- 

151 filename : convertible to `ResourcePath` 

152 Path or URI to the override file. All URI schemes supported by 

153 `ResourcePath` are supported. 

154 """ 

155 self._overrides.append((OverrideTypes.File, ResourcePath(filename))) 

156 

157 def addValueOverride(self, field, value): 

158 """Add override for a specific field. 

159 

160 This method is not very type-safe as it is designed to support 

161 use cases where input is given as string, e.g. command line 

162 activators. If `value` has a string type and setting of the field 

163 fails with `TypeError` the we'll attempt `eval()` the value and 

164 set the field with that value instead. 

165 

166 Parameters 

167 ---------- 

168 field : str 

169 Fully-qualified field name. 

170 value : 

171 Value to be given to a filed. 

172 """ 

173 self._overrides.append((OverrideTypes.Value, (field, value))) 

174 

175 def addPythonOverride(self, python_snippet: str) -> None: 

176 """Add Overrides by running a snippit of python code against a config. 

177 

178 Parameters 

179 ---------- 

180 python_snippet: str 

181 A string which is valid python code to be executed. This is done 

182 with config as the only local accessible value. 

183 """ 

184 self._overrides.append((OverrideTypes.Python, python_snippet)) 

185 

186 def addInstrumentOverride(self, instrument: str, task_name: str) -> None: 

187 """Apply any overrides that an instrument has for a task 

188 

189 Parameters 

190 ---------- 

191 instrument: str 

192 A string containing the fully qualified name of an instrument from 

193 which configs should be loaded and applied 

194 task_name: str 

195 The _DefaultName of a task associated with a config, used to look 

196 up overrides from the instrument. 

197 """ 

198 instrument_cls: type = doImportType(instrument) 

199 instrument_lib = instrument_cls() 

200 self._overrides.append((OverrideTypes.Instrument, (instrument_lib, task_name))) 

201 

202 def _parser(self, value, configParser): 

203 try: 

204 value = configParser.visit(ast.parse(value, mode="eval").body) 

205 except Exception: 

206 # This probably means it is a specific user string such as a URI. 

207 # Let the value return as a string to attempt to continue to 

208 # process as a string, another error will be raised in downstream 

209 # code if that assumption is wrong 

210 pass 

211 

212 return value 

213 

214 def applyTo(self, config): 

215 """Apply all overrides to a task configuration object. 

216 

217 Parameters 

218 ---------- 

219 config : `pex.Config` 

220 Configuration to apply to; modified in place. 

221 

222 Raises 

223 ------ 

224 `Exception` is raised if operations on configuration object fail. 

225 """ 

226 # Look up a stack of variables people may be using when setting 

227 # configs. Create a dictionary that will be used akin to a namespace 

228 # for the duration of this function. 

229 vars = {} 

230 # pull in the variables that are declared in module scope of the config 

231 mod = inspect.getmodule(config) 

232 vars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

233 # put the supplied config in the variables dictionary 

234 vars["config"] = config 

235 

236 # Create a parser for config expressions that may be strings 

237 configParser = ConfigExpressionParser(namespace=vars) 

238 

239 for otype, override in self._overrides: 

240 if otype is OverrideTypes.File: 

241 with override.open("r") as buffer: 

242 config.loadFromStream(buffer, filename=override.ospath) 

243 elif otype is OverrideTypes.Value: 

244 field, value = override 

245 if isinstance(value, str): 

246 value = self._parser(value, configParser) 

247 # checking for dicts and lists here is needed because {} and [] 

248 # are valid yaml syntax so they get converted before hitting 

249 # this method, so we must parse the elements. 

250 # 

251 # The same override would remain a string if specified on the 

252 # command line, and is handled above. 

253 if isinstance(value, dict): 

254 new = {} 

255 for k, v in value.items(): 

256 if isinstance(v, str): 

257 new[k] = self._parser(v, configParser) 

258 else: 

259 new[k] = v 

260 value = new 

261 elif isinstance(value, list): 

262 new = [] 

263 for v in value: 

264 if isinstance(v, str): 

265 new.append(self._parser(v, configParser)) 

266 else: 

267 new.append(v) 

268 value = new 

269 # The field might be a string corresponding to a attribute 

270 # hierarchy, attempt to split off the last field which 

271 # will then be set. 

272 parent, *child = field.rsplit(".", maxsplit=1) 

273 if child: 

274 # This branch means there was a hierarchy, get the 

275 # field to set, and look up the sub config for which 

276 # it is to be set 

277 finalField = child[0] 

278 tmpConfig = attrgetter(parent)(config) 

279 else: 

280 # There is no hierarchy, the config is the base config 

281 # and the field is exactly what was passed in 

282 finalField = parent 

283 tmpConfig = config 

284 # set the specified config 

285 setattr(tmpConfig, finalField, value) 

286 

287 elif otype is OverrideTypes.Python: 

288 # exec python string with the context of all vars known. This 

289 # both lets people use a var they know about (maybe a bit 

290 # dangerous, but not really more so than arbitrary python exec, 

291 # and there are so many other places to worry about security 

292 # before we start changing this) and any imports that are done 

293 # in a python block will be put into this scope. This means 

294 # other config setting branches can make use of these 

295 # variables. 

296 exec(override, None, vars) 

297 elif otype is OverrideTypes.Instrument: 

298 instrument, name = override 

299 instrument.applyConfigOverrides(name, config)