Coverage for python/lsst/pipe/base/configOverrides.py: 22%

112 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-11 10:50 +0000

1# This file is part of pipe_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28"""Module which defines ConfigOverrides class and related methods. 

29""" 

30from __future__ import annotations 

31 

32__all__ = ["ConfigOverrides"] 

33 

34import ast 

35import contextlib 

36import inspect 

37from enum import Enum 

38from operator import attrgetter 

39from types import SimpleNamespace 

40from typing import TYPE_CHECKING, Any 

41 

42from lsst.resources import ResourcePath 

43 

44from ._instrument import Instrument 

45 

46if TYPE_CHECKING: 

47 from .pipelineIR import ParametersIR 

48 

49OverrideTypes = Enum("OverrideTypes", "Value File Python Instrument") 

50 

51 

52class _FrozenSimpleNamespace(SimpleNamespace): 

53 """SimpleNamespace subclass which disallows setting after construction""" 

54 

55 def __init__(self, **kwargs: Any) -> None: 

56 object.__setattr__(self, "_frozen", False) 

57 super().__init__(**kwargs) 

58 self._frozen = True 

59 

60 def __setattr__(self, __name: str, __value: Any) -> None: 

61 if self._frozen: 

62 raise ValueError("Cannot set attributes on parameters") 

63 else: 

64 return super().__setattr__(__name, __value) 

65 

66 

67class ConfigExpressionParser(ast.NodeVisitor): 

68 """An expression parser that will be used to transform configuration 

69 strings supplied from the command line or a pipeline into a python 

70 object. 

71 

72 This is roughly equivalent to ast.literal_parser, but with the ability to 

73 transform strings that are valid variable names into the value associated 

74 with the name. Variables that should be considered valid are supplied to 

75 the constructor as a dictionary that maps a string to its corresponding 

76 value. 

77 

78 This class in an internal implementation detail, and should not be exposed 

79 outside this module. 

80 

81 Parameters 

82 ---------- 

83 namespace : `dict` of `str` to variable 

84 This is a mapping of strings corresponding to variable names, to the 

85 object that is associated with that name. 

86 """ 

87 

88 def __init__(self, namespace): 

89 self.variables = namespace 

90 

91 def visit_Name(self, node): 

92 """Handle a node corresponding to a variable name. 

93 

94 Parameters 

95 ---------- 

96 node : `ast.Name` 

97 Node to use. 

98 """ 

99 # If the id (name) of the variable is in the dictionary of valid names, 

100 # load and return the corresponding variable. 

101 if node.id in self.variables: 

102 return self.variables[node.id] 

103 # If the node does not correspond to a valid variable, turn the name 

104 # into a string, as the user likely intended it as such. 

105 return f"{node.id}" 

106 

107 def visit_List(self, node): 

108 """Build a list out of the sub nodes when a list node is 

109 encountered. 

110 

111 Parameters 

112 ---------- 

113 node : `ast.Name` 

114 Node to use. 

115 """ 

116 return [self.visit(elm) for elm in node.elts] 

117 

118 def visit_Tuple(self, node): 

119 """Build a list out of the sub nodes and then turn it into a 

120 tuple. 

121 

122 Parameters 

123 ---------- 

124 node : `ast.Name` 

125 Node to use. 

126 """ 

127 return tuple(self.visit_List(node)) 

128 

129 def visit_Constant(self, node): 

130 """Return constant from node. 

131 

132 Parameters 

133 ---------- 

134 node : `ast.Name` 

135 Node to use. 

136 """ 

137 return node.value 

138 

139 def visit_Dict(self, node): 

140 """Build dict out of component nodes if dict node encountered. 

141 

142 Parameters 

143 ---------- 

144 node : `ast.Name` 

145 Node to use. 

146 """ 

147 return {self.visit(key): self.visit(value) for key, value in zip(node.keys, node.values, strict=True)} 

148 

149 def visit_Set(self, node): 

150 """Build set out of node is set encountered. 

151 

152 Parameters 

153 ---------- 

154 node : `ast.Name` 

155 Node to use. 

156 """ 

157 return {self.visit(el) for el in node.elts} 

158 

159 def visit_UnaryOp(self, node): 

160 """Handle unary operators. 

161 

162 This method is visited if the node is a unary operator. Currently 

163 The only operator we support is the negative (-) operator, all others 

164 are passed to generic_visit method. 

165 

166 Parameters 

167 ---------- 

168 node : `ast.Name` 

169 Node to use. 

170 """ 

171 if isinstance(node.op, ast.USub): 

172 value = self.visit(node.operand) 

173 return -1 * value 

174 self.generic_visit(node) 

175 

176 def generic_visit(self, node): 

177 """Handle other node types. 

178 

179 This method is called for all other node types. It will just raise 

180 a value error, because this is a type of expression that we do not 

181 support. 

182 

183 Parameters 

184 ---------- 

185 node : `ast.Name` 

186 Node to use. 

187 """ 

188 raise ValueError("Unable to parse string into literal expression") 

189 

190 

191class ConfigOverrides: 

192 """Defines a set of overrides to be applied to a task config. 

193 

194 Overrides for task configuration need to be applied by activator when 

195 creating task instances. This class represents an ordered set of such 

196 overrides which activator receives from some source (e.g. command line 

197 or some other configuration). 

198 

199 Methods 

200 ------- 

201 addFileOverride(filename) 

202 Add overrides from a specified file. 

203 addValueOverride(field, value) 

204 Add override for a specific field. 

205 applyTo(config) 

206 Apply all overrides to a `config` instance. 

207 

208 Notes 

209 ----- 

210 Serialization support for this class may be needed, will add later if 

211 necessary. 

212 """ 

213 

214 def __init__(self) -> None: 

215 self._overrides: list[tuple[OverrideTypes, Any]] = [] 

216 self._parameters: SimpleNamespace | None = None 

217 

218 def addParameters(self, parameters: ParametersIR) -> None: 

219 """Add parameters which will be substituted when applying overrides. 

220 

221 Parameters 

222 ---------- 

223 parameters : `ParametersIR` 

224 Override parameters in the form as read from a Pipeline file. 

225 

226 Notes 

227 ----- 

228 This method may be called more than once, but each call will overwrite 

229 any previous parameter defined with the same name. 

230 """ 

231 if self._parameters is None: 

232 self._parameters = SimpleNamespace() 

233 

234 for key, value in parameters.mapping.items(): 

235 setattr(self._parameters, key, value) 

236 

237 def addFileOverride(self, filename): 

238 """Add overrides from a specified file. 

239 

240 Parameters 

241 ---------- 

242 filename : convertible to `~lsst.resources.ResourcePath` 

243 Path or URI to the override file. All URI schemes supported by 

244 `~lsst.resources.ResourcePath` are supported. 

245 """ 

246 self._overrides.append((OverrideTypes.File, ResourcePath(filename))) 

247 

248 def addValueOverride(self, field, value): 

249 """Add override for a specific field. 

250 

251 This method is not very type-safe as it is designed to support 

252 use cases where input is given as string, e.g. command line 

253 activators. If `value` has a string type and setting of the field 

254 fails with `TypeError` the we'll attempt `eval()` the value and 

255 set the field with that value instead. 

256 

257 Parameters 

258 ---------- 

259 field : str 

260 Fully-qualified field name. 

261 value : `~typing.Any` 

262 Value to be given to a filed. 

263 """ 

264 self._overrides.append((OverrideTypes.Value, (field, value))) 

265 

266 def addPythonOverride(self, python_snippet: str) -> None: 

267 """Add Overrides by running a snippit of python code against a config. 

268 

269 Parameters 

270 ---------- 

271 python_snippet : str 

272 A string which is valid python code to be executed. This is done 

273 with config as the only local accessible value. 

274 """ 

275 self._overrides.append((OverrideTypes.Python, python_snippet)) 

276 

277 def addInstrumentOverride(self, instrument: Instrument, task_name: str) -> None: 

278 """Apply any overrides that an instrument has for a task. 

279 

280 Parameters 

281 ---------- 

282 instrument : `Instrument` 

283 An instrument instance which will apply configs. 

284 task_name : str 

285 The _DefaultName of a task associated with a config, used to look 

286 up overrides from the instrument. 

287 """ 

288 self._overrides.append((OverrideTypes.Instrument, (instrument, task_name))) 

289 

290 def _parser(self, value, configParser): 

291 # Exception probably means it is a specific user string such as a URI. 

292 # Let the value return as a string to attempt to continue to 

293 # process as a string, another error will be raised in downstream 

294 # code if that assumption is wrong 

295 with contextlib.suppress(Exception): 

296 value = configParser.visit(ast.parse(value, mode="eval").body) 

297 return value 

298 

299 def applyTo(self, config): 

300 """Apply all overrides to a task configuration object. 

301 

302 Parameters 

303 ---------- 

304 config : `pex.Config` 

305 Configuration to apply to; modified in place. 

306 

307 Raises 

308 ------ 

309 `Exception` is raised if operations on configuration object fail. 

310 """ 

311 # Look up a stack of variables people may be using when setting 

312 # configs. Create a dictionary that will be used akin to a namespace 

313 # for the duration of this function. 

314 localVars = {} 

315 # pull in the variables that are declared in module scope of the config 

316 mod = inspect.getmodule(config) 

317 localVars.update({k: v for k, v in mod.__dict__.items() if not k.startswith("__")}) 

318 # put the supplied config in the variables dictionary 

319 localVars["config"] = config 

320 extraLocals = None 

321 

322 # If any parameters are supplied add them to the variables dictionary 

323 if self._parameters is not None: 

324 # make a copy of the params and "freeze" it 

325 localParams = _FrozenSimpleNamespace(**vars(self._parameters)) 

326 localVars["parameters"] = localParams 

327 extraLocals = {"parameters": localParams} 

328 

329 # Create a parser for config expressions that may be strings 

330 configParser = ConfigExpressionParser(namespace=localVars) 

331 

332 for otype, override in self._overrides: 

333 if otype is OverrideTypes.File: 

334 with override.open("r") as buffer: 

335 config.loadFromStream(buffer, filename=override.ospath, extraLocals=extraLocals) 

336 elif otype is OverrideTypes.Value: 

337 field, value = override 

338 if isinstance(value, str): 

339 value = self._parser(value, configParser) 

340 # checking for dicts and lists here is needed because {} and [] 

341 # are valid yaml syntax so they get converted before hitting 

342 # this method, so we must parse the elements. 

343 # 

344 # The same override would remain a string if specified on the 

345 # command line, and is handled above. 

346 if isinstance(value, dict): 

347 new = {} 

348 for k, v in value.items(): 

349 if isinstance(v, str): 

350 new[k] = self._parser(v, configParser) 

351 else: 

352 new[k] = v 

353 value = new 

354 elif isinstance(value, list): 

355 new = [] 

356 for v in value: 

357 if isinstance(v, str): 

358 new.append(self._parser(v, configParser)) 

359 else: 

360 new.append(v) 

361 value = new 

362 # The field might be a string corresponding to a attribute 

363 # hierarchy, attempt to split off the last field which 

364 # will then be set. 

365 parent, *child = field.rsplit(".", maxsplit=1) 

366 if child: 

367 # This branch means there was a hierarchy, get the 

368 # field to set, and look up the sub config for which 

369 # it is to be set 

370 finalField = child[0] 

371 tmpConfig = attrgetter(parent)(config) 

372 else: 

373 # There is no hierarchy, the config is the base config 

374 # and the field is exactly what was passed in 

375 finalField = parent 

376 tmpConfig = config 

377 # set the specified config 

378 setattr(tmpConfig, finalField, value) 

379 

380 elif otype is OverrideTypes.Python: 

381 # exec python string with the context of all vars known. This 

382 # both lets people use a var they know about (maybe a bit 

383 # dangerous, but not really more so than arbitrary python exec, 

384 # and there are so many other places to worry about security 

385 # before we start changing this) and any imports that are done 

386 # in a python block will be put into this scope. This means 

387 # other config setting branches can make use of these 

388 # variables. 

389 exec(override, None, localVars) 

390 elif otype is OverrideTypes.Instrument: 

391 instrument, name = override 

392 instrument.applyConfigOverrides(name, config)