Coverage for python/lsst/pipe/base/configOverrides.py: 22%

112 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-09-19 10:39 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Module which defines ConfigOverrides class and related methods. 

29""" 

30from __future__ import annotations 

31 

32__all__ = ["ConfigOverrides"] 

33 

34import ast 

35import contextlib 

36import inspect 

37from enum import Enum 

38from operator import attrgetter 

39from types import SimpleNamespace 

40from typing import TYPE_CHECKING, Any 

41 

42from lsst.resources import ResourcePath 

43 

44from ._instrument import Instrument 

45 

46if TYPE_CHECKING: 

47 from .pipelineIR import ParametersIR 

48 

49OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

50 

51 

52class _FrozenSimpleNamespace(SimpleNamespace): 

53 """SimpleNamespace subclass which disallows setting after construction""" 

54 

55 def __init__(self, **kwargs: Any) -> None: 

56 object.__setattr__(self, "_frozen", False) 

57 super().__init__(**kwargs) 

58 self._frozen = True 

59 

60 def __setattr__(self, __name: str, __value: Any) -> None: 

61 if self._frozen: 

62 raise ValueError("Cannot set attributes on parameters") 

63 else: 

64 return super().__setattr__(__name, __value) 

65 

66 

67class ConfigExpressionParser(ast.NodeVisitor): 

68 """An expression parser that will be used to transform configuration 

69 strings supplied from the command line or a pipeline into a python 

70 object. 

71 

72 This is roughly equivalent to ast.literal_parser, but with the ability to 

73 transform strings that are valid variable names into the value associated 

74 with the name. Variables that should be considered valid are supplied to 

75 the constructor as a dictionary that maps a string to its corresponding 

76 value. 

77 

78 This class in an internal implementation detail, and should not be exposed 

79 outside this module. 

80 

81 Parameters 

82 ---------- 

83 namespace : `dict` of `str` to variable 

84 This is a mapping of strings corresponding to variable names, to the 

85 object that is associated with that name 

86 """ 

87 

88 def __init__(self, namespace): 

89 self.variables = namespace 

90 

91 def visit_Name(self, node): 

92 """Handle a node corresponding to a variable name.""" 

93 # If the id (name) of the variable is in the dictionary of valid names, 

94 # load and return the corresponding variable. 

95 if node.id in self.variables: 

96 return self.variables[node.id] 

97 # If the node does not correspond to a valid variable, turn the name 

98 # into a string, as the user likely intended it as such. 

99 return f"{node.id}" 

100 

101 def visit_List(self, node): 

102 """Build a list out of the sub nodes when a list node is 

103 encountered. 

104 """ 

105 return [self.visit(elm) for elm in node.elts] 

106 

107 def visit_Tuple(self, node): 

108 """Build a list out of the sub nodes and then turn it into a 

109 tuple. 

110 """ 

111 return tuple(self.visit_List(node)) 

112 

113 def visit_Constant(self, node): 

114 """Return constant from node.""" 

115 return node.value 

116 

117 def visit_Dict(self, node): 

118 """Build dict out of component nodes if dict node encountered.""" 

119 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values, strict=True)} 

120 

121 def visit_Set(self, node): 

122 """Build set out of node is set encountered.""" 

123 return {self.visit(el) for el in node.elts} 

124 

125 def visit_UnaryOp(self, node): 

126 """Handle unary operators. 

127 

128 This method is visited if the node is a unary operator. Currently 

129 The only operator we support is the negative (-) operator, all others 

130 are passed to generic_visit method. 

131 """ 

132 if isinstance(node.op, ast.USub): 

133 value = self.visit(node.operand) 

134 return -1 * value 

135 self.generic_visit(node) 

136 

137 def generic_visit(self, node): 

138 """Handle other node types. 

139 

140 This method is called for all other node types. It will just raise 

141 a value error, because this is a type of expression that we do not 

142 support. 

143 """ 

144 raise ValueError("Unable to parse string into literal expression") 

145 

146 

147class ConfigOverrides: 

148 """Defines a set of overrides to be applied to a task config. 

149 

150 Overrides for task configuration need to be applied by activator when 

151 creating task instances. This class represents an ordered set of such 

152 overrides which activator receives from some source (e.g. command line 

153 or some other configuration). 

154 

155 Methods 

156 ------- 

157 addFileOverride(filename) 

158 Add overrides from a specified file. 

159 addValueOverride(field, value) 

160 Add override for a specific field. 

161 applyTo(config) 

162 Apply all overrides to a `config` instance. 

163 

164 Notes 

165 ----- 

166 Serialization support for this class may be needed, will add later if 

167 necessary. 

168 """ 

169 

170 def __init__(self) -> None: 

171 self._overrides: list[tuple[OverrideTypes, Any]] = [] 

172 self._parameters: SimpleNamespace | None = None 

173 

174 def addParameters(self, parameters: ParametersIR) -> None: 

175 """Add parameters which will be substituted when applying overrides. 

176 

177 Parameters 

178 ---------- 

179 parameters : `ParametersIR` 

180 Override parameters in the form as read from a Pipeline file. 

181 

182 Note 

183 ---- 

184 This method may be called more than once, but each call will overwrite 

185 any previous parameter defined with the same name. 

186 """ 

187 if self._parameters is None: 

188 self._parameters = SimpleNamespace() 

189 

190 for key, value in parameters.mapping.items(): 

191 setattr(self._parameters, key, value) 

192 

193 def addFileOverride(self, filename): 

194 """Add overrides from a specified file. 

195 

196 Parameters 

197 ---------- 

198 filename : convertible to `~lsst.resources.ResourcePath` 

199 Path or URI to the override file. All URI schemes supported by 

200 `~lsst.resources.ResourcePath` are supported. 

201 """ 

202 self._overrides.append((OverrideTypes.File, ResourcePath(filename))) 

203 

204 def addValueOverride(self, field, value): 

205 """Add override for a specific field. 

206 

207 This method is not very type-safe as it is designed to support 

208 use cases where input is given as string, e.g. command line 

209 activators. If `value` has a string type and setting of the field 

210 fails with `TypeError` the we'll attempt `eval()` the value and 

211 set the field with that value instead. 

212 

213 Parameters 

214 ---------- 

215 field : str 

216 Fully-qualified field name. 

217 value : 

218 Value to be given to a filed. 

219 """ 

220 self._overrides.append((OverrideTypes.Value, (field, value))) 

221 

222 def addPythonOverride(self, python_snippet: str) -> None: 

223 """Add Overrides by running a snippit of python code against a config. 

224 

225 Parameters 

226 ---------- 

227 python_snippet: str 

228 A string which is valid python code to be executed. This is done 

229 with config as the only local accessible value. 

230 """ 

231 self._overrides.append((OverrideTypes.Python, python_snippet)) 

232 

233 def addInstrumentOverride(self, instrument: Instrument, task_name: str) -> None: 

234 """Apply any overrides that an instrument has for a task 

235 

236 Parameters 

237 ---------- 

238 instrument: `Instrument` 

239 An instrument instance which will apply configs 

240 task_name: str 

241 The _DefaultName of a task associated with a config, used to look 

242 up overrides from the instrument. 

243 """ 

244 self._overrides.append((OverrideTypes.Instrument, (instrument, task_name))) 

245 

246 def _parser(self, value, configParser): 

247 # Exception probably means it is a specific user string such as a URI. 

248 # Let the value return as a string to attempt to continue to 

249 # process as a string, another error will be raised in downstream 

250 # code if that assumption is wrong 

251 with contextlib.suppress(Exception): 

252 value = configParser.visit(ast.parse(value, mode="eval").body) 

253 return value 

254 

255 def applyTo(self, config): 

256 """Apply all overrides to a task configuration object. 

257 

258 Parameters 

259 ---------- 

260 config : `pex.Config` 

261 Configuration to apply to; modified in place. 

262 

263 Raises 

264 ------ 

265 `Exception` is raised if operations on configuration object fail. 

266 """ 

267 # Look up a stack of variables people may be using when setting 

268 # configs. Create a dictionary that will be used akin to a namespace 

269 # for the duration of this function. 

270 localVars = {} 

271 # pull in the variables that are declared in module scope of the config 

272 mod = inspect.getmodule(config) 

273 localVars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

274 # put the supplied config in the variables dictionary 

275 localVars["config"] = config 

276 extraLocals = None 

277 

278 # If any parameters are supplied add them to the variables dictionary 

279 if self._parameters is not None: 

280 # make a copy of the params and "freeze" it 

281 localParams = _FrozenSimpleNamespace(**vars(self._parameters)) 

282 localVars["parameters"] = localParams 

283 extraLocals = {"parameters": localParams} 

284 

285 # Create a parser for config expressions that may be strings 

286 configParser = ConfigExpressionParser(namespace=localVars) 

287 

288 for otype, override in self._overrides: 

289 if otype is OverrideTypes.File: 

290 with override.open("r") as buffer: 

291 config.loadFromStream(buffer, filename=override.ospath, extraLocals=extraLocals) 

292 elif otype is OverrideTypes.Value: 

293 field, value = override 

294 if isinstance(value, str): 

295 value = self._parser(value, configParser) 

296 # checking for dicts and lists here is needed because {} and [] 

297 # are valid yaml syntax so they get converted before hitting 

298 # this method, so we must parse the elements. 

299 # 

300 # The same override would remain a string if specified on the 

301 # command line, and is handled above. 

302 if isinstance(value, dict): 

303 new = {} 

304 for k, v in value.items(): 

305 if isinstance(v, str): 

306 new[k] = self._parser(v, configParser) 

307 else: 

308 new[k] = v 

309 value = new 

310 elif isinstance(value, list): 

311 new = [] 

312 for v in value: 

313 if isinstance(v, str): 

314 new.append(self._parser(v, configParser)) 

315 else: 

316 new.append(v) 

317 value = new 

318 # The field might be a string corresponding to a attribute 

319 # hierarchy, attempt to split off the last field which 

320 # will then be set. 

321 parent, *child = field.rsplit(".", maxsplit=1) 

322 if child: 

323 # This branch means there was a hierarchy, get the 

324 # field to set, and look up the sub config for which 

325 # it is to be set 

326 finalField = child[0] 

327 tmpConfig = attrgetter(parent)(config) 

328 else: 

329 # There is no hierarchy, the config is the base config 

330 # and the field is exactly what was passed in 

331 finalField = parent 

332 tmpConfig = config 

333 # set the specified config 

334 setattr(tmpConfig, finalField, value) 

335 

336 elif otype is OverrideTypes.Python: 

337 # exec python string with the context of all vars known. This 

338 # both lets people use a var they know about (maybe a bit 

339 # dangerous, but not really more so than arbitrary python exec, 

340 # and there are so many other places to worry about security 

341 # before we start changing this) and any imports that are done 

342 # in a python block will be put into this scope. This means 

343 # other config setting branches can make use of these 

344 # variables. 

345 exec(override, None, localVars) 

346 elif otype is OverrideTypes.Instrument: 

347 instrument, name = override 

348 instrument.applyConfigOverrides(name, config)