Coverage for python/lsst/pipe/base/configOverrides.py: 22%

112 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-31 09:39 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22"""Module which defines ConfigOverrides class and related methods. 

23""" 

24from __future__ import annotations 

25 

26__all__ = ["ConfigOverrides"] 

27 

28import ast 

29import contextlib 

30import inspect 

31from enum import Enum 

32from operator import attrgetter 

33from types import SimpleNamespace 

34from typing import TYPE_CHECKING, Any 

35 

36from lsst.resources import ResourcePath 

37 

38from ._instrument import Instrument 

39 

40if TYPE_CHECKING: 

41 from .pipelineIR import ParametersIR 

42 

43OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

44 

45 

46class _FrozenSimpleNamespace(SimpleNamespace): 

47 """SimpleNamespace subclass which disallows setting after construction""" 

48 

49 def __init__(self, **kwargs: Any) -> None: 

50 object.__setattr__(self, "_frozen", False) 

51 super().__init__(**kwargs) 

52 self._frozen = True 

53 

54 def __setattr__(self, __name: str, __value: Any) -> None: 

55 if self._frozen: 

56 raise ValueError("Cannot set attributes on parameters") 

57 else: 

58 return super().__setattr__(__name, __value) 

59 

60 

61class ConfigExpressionParser(ast.NodeVisitor): 

62 """An expression parser that will be used to transform configuration 

63 strings supplied from the command line or a pipeline into a python 

64 object. 

65 

66 This is roughly equivalent to ast.literal_parser, but with the ability to 

67 transform strings that are valid variable names into the value associated 

68 with the name. Variables that should be considered valid are supplied to 

69 the constructor as a dictionary that maps a string to its corresponding 

70 value. 

71 

72 This class in an internal implementation detail, and should not be exposed 

73 outside this module. 

74 

75 Parameters 

76 ---------- 

77 namespace : `dict` of `str` to variable 

78 This is a mapping of strings corresponding to variable names, to the 

79 object that is associated with that name 

80 """ 

81 

82 def __init__(self, namespace): 

83 self.variables = namespace 

84 

85 def visit_Name(self, node): 

86 """Handle a node corresponding to a variable name.""" 

87 # If the id (name) of the variable is in the dictionary of valid names, 

88 # load and return the corresponding variable. 

89 if node.id in self.variables: 

90 return self.variables[node.id] 

91 # If the node does not correspond to a valid variable, turn the name 

92 # into a string, as the user likely intended it as such. 

93 return f"{node.id}" 

94 

95 def visit_List(self, node): 

96 """Build a list out of the sub nodes when a list node is 

97 encountered. 

98 """ 

99 return [self.visit(elm) for elm in node.elts] 

100 

101 def visit_Tuple(self, node): 

102 """Build a list out of the sub nodes and then turn it into a 

103 tuple. 

104 """ 

105 return tuple(self.visit_List(node)) 

106 

107 def visit_Constant(self, node): 

108 """Return constant from node.""" 

109 return node.value 

110 

111 def visit_Dict(self, node): 

112 """Build dict out of component nodes if dict node encountered.""" 

113 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values, strict=True)} 

114 

115 def visit_Set(self, node): 

116 """Build set out of node is set encountered.""" 

117 return {self.visit(el) for el in node.elts} 

118 

119 def visit_UnaryOp(self, node): 

120 """Handle unary operators. 

121 

122 This method is visited if the node is a unary operator. Currently 

123 The only operator we support is the negative (-) operator, all others 

124 are passed to generic_visit method. 

125 """ 

126 if isinstance(node.op, ast.USub): 

127 value = self.visit(node.operand) 

128 return -1 * value 

129 self.generic_visit(node) 

130 

131 def generic_visit(self, node): 

132 """Handle other node types. 

133 

134 This method is called for all other node types. It will just raise 

135 a value error, because this is a type of expression that we do not 

136 support. 

137 """ 

138 raise ValueError("Unable to parse string into literal expression") 

139 

140 

141class ConfigOverrides: 

142 """Defines a set of overrides to be applied to a task config. 

143 

144 Overrides for task configuration need to be applied by activator when 

145 creating task instances. This class represents an ordered set of such 

146 overrides which activator receives from some source (e.g. command line 

147 or some other configuration). 

148 

149 Methods 

150 ------- 

151 addFileOverride(filename) 

152 Add overrides from a specified file. 

153 addValueOverride(field, value) 

154 Add override for a specific field. 

155 applyTo(config) 

156 Apply all overrides to a `config` instance. 

157 

158 Notes 

159 ----- 

160 Serialization support for this class may be needed, will add later if 

161 necessary. 

162 """ 

163 

164 def __init__(self) -> None: 

165 self._overrides: list[tuple[OverrideTypes, Any]] = [] 

166 self._parameters: SimpleNamespace | None = None 

167 

168 def addParameters(self, parameters: ParametersIR) -> None: 

169 """Add parameters which will be substituted when applying overrides. 

170 

171 Parameters 

172 ---------- 

173 parameters : `ParametersIR` 

174 Override parameters in the form as read from a Pipeline file. 

175 

176 Note 

177 ---- 

178 This method may be called more than once, but each call will overwrite 

179 any previous parameter defined with the same name. 

180 """ 

181 if self._parameters is None: 

182 self._parameters = SimpleNamespace() 

183 

184 for key, value in parameters.mapping.items(): 

185 setattr(self._parameters, key, value) 

186 

187 def addFileOverride(self, filename): 

188 """Add overrides from a specified file. 

189 

190 Parameters 

191 ---------- 

192 filename : convertible to `~lsst.resources.ResourcePath` 

193 Path or URI to the override file. All URI schemes supported by 

194 `~lsst.resources.ResourcePath` are supported. 

195 """ 

196 self._overrides.append((OverrideTypes.File, ResourcePath(filename))) 

197 

198 def addValueOverride(self, field, value): 

199 """Add override for a specific field. 

200 

201 This method is not very type-safe as it is designed to support 

202 use cases where input is given as string, e.g. command line 

203 activators. If `value` has a string type and setting of the field 

204 fails with `TypeError` the we'll attempt `eval()` the value and 

205 set the field with that value instead. 

206 

207 Parameters 

208 ---------- 

209 field : str 

210 Fully-qualified field name. 

211 value : 

212 Value to be given to a filed. 

213 """ 

214 self._overrides.append((OverrideTypes.Value, (field, value))) 

215 

216 def addPythonOverride(self, python_snippet: str) -> None: 

217 """Add Overrides by running a snippit of python code against a config. 

218 

219 Parameters 

220 ---------- 

221 python_snippet: str 

222 A string which is valid python code to be executed. This is done 

223 with config as the only local accessible value. 

224 """ 

225 self._overrides.append((OverrideTypes.Python, python_snippet)) 

226 

227 def addInstrumentOverride(self, instrument: Instrument, task_name: str) -> None: 

228 """Apply any overrides that an instrument has for a task 

229 

230 Parameters 

231 ---------- 

232 instrument: `Instrument` 

233 An instrument instance which will apply configs 

234 task_name: str 

235 The _DefaultName of a task associated with a config, used to look 

236 up overrides from the instrument. 

237 """ 

238 self._overrides.append((OverrideTypes.Instrument, (instrument, task_name))) 

239 

240 def _parser(self, value, configParser): 

241 # Exception probably means it is a specific user string such as a URI. 

242 # Let the value return as a string to attempt to continue to 

243 # process as a string, another error will be raised in downstream 

244 # code if that assumption is wrong 

245 with contextlib.suppress(Exception): 

246 value = configParser.visit(ast.parse(value, mode="eval").body) 

247 return value 

248 

249 def applyTo(self, config): 

250 """Apply all overrides to a task configuration object. 

251 

252 Parameters 

253 ---------- 

254 config : `pex.Config` 

255 Configuration to apply to; modified in place. 

256 

257 Raises 

258 ------ 

259 `Exception` is raised if operations on configuration object fail. 

260 """ 

261 # Look up a stack of variables people may be using when setting 

262 # configs. Create a dictionary that will be used akin to a namespace 

263 # for the duration of this function. 

264 localVars = {} 

265 # pull in the variables that are declared in module scope of the config 

266 mod = inspect.getmodule(config) 

267 localVars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

268 # put the supplied config in the variables dictionary 

269 localVars["config"] = config 

270 extraLocals = None 

271 

272 # If any parameters are supplied add them to the variables dictionary 

273 if self._parameters is not None: 

274 # make a copy of the params and "freeze" it 

275 localParams = _FrozenSimpleNamespace(**vars(self._parameters)) 

276 localVars["parameters"] = localParams 

277 extraLocals = {"parameters": localParams} 

278 

279 # Create a parser for config expressions that may be strings 

280 configParser = ConfigExpressionParser(namespace=localVars) 

281 

282 for otype, override in self._overrides: 

283 if otype is OverrideTypes.File: 

284 with override.open("r") as buffer: 

285 config.loadFromStream(buffer, filename=override.ospath, extraLocals=extraLocals) 

286 elif otype is OverrideTypes.Value: 

287 field, value = override 

288 if isinstance(value, str): 

289 value = self._parser(value, configParser) 

290 # checking for dicts and lists here is needed because {} and [] 

291 # are valid yaml syntax so they get converted before hitting 

292 # this method, so we must parse the elements. 

293 # 

294 # The same override would remain a string if specified on the 

295 # command line, and is handled above. 

296 if isinstance(value, dict): 

297 new = {} 

298 for k, v in value.items(): 

299 if isinstance(v, str): 

300 new[k] = self._parser(v, configParser) 

301 else: 

302 new[k] = v 

303 value = new 

304 elif isinstance(value, list): 

305 new = [] 

306 for v in value: 

307 if isinstance(v, str): 

308 new.append(self._parser(v, configParser)) 

309 else: 

310 new.append(v) 

311 value = new 

312 # The field might be a string corresponding to a attribute 

313 # hierarchy, attempt to split off the last field which 

314 # will then be set. 

315 parent, *child = field.rsplit(".", maxsplit=1) 

316 if child: 

317 # This branch means there was a hierarchy, get the 

318 # field to set, and look up the sub config for which 

319 # it is to be set 

320 finalField = child[0] 

321 tmpConfig = attrgetter(parent)(config) 

322 else: 

323 # There is no hierarchy, the config is the base config 

324 # and the field is exactly what was passed in 

325 finalField = parent 

326 tmpConfig = config 

327 # set the specified config 

328 setattr(tmpConfig, finalField, value) 

329 

330 elif otype is OverrideTypes.Python: 

331 # exec python string with the context of all vars known. This 

332 # both lets people use a var they know about (maybe a bit 

333 # dangerous, but not really more so than arbitrary python exec, 

334 # and there are so many other places to worry about security 

335 # before we start changing this) and any imports that are done 

336 # in a python block will be put into this scope. This means 

337 # other config setting branches can make use of these 

338 # variables. 

339 exec(override, None, localVars) 

340 elif otype is OverrideTypes.Instrument: 

341 instrument, name = override 

342 instrument.applyConfigOverrides(name, config)