Coverage for python/lsst/ctrl/bps/bps_config.py: 12%

132 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-10 19:09 +0000

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <https://www.gnu.org/licenses/>. 

27 

28"""Configuration class that adds order to searching sections for value, 

29expands environment variables and other config variables. 

30""" 

31 

32__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

33 

34 

35import copy 

36import logging 

37import re 

38import string 

39from os.path import expandvars 

40 

41from lsst.daf.butler import Config 

42from lsst.resources import ResourcePath 

43 

44_LOG = logging.getLogger(__name__) 

45 

46# Using lsst.daf.butler.Config to resolve possible includes. 

47BPS_DEFAULTS = Config(ResourcePath("resource://lsst.ctrl.bps/etc/bps_defaults.yaml")).toDict() 

48 

49BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "cloud", "bps_defined"] 

50 

51# Need a string that won't be a valid default value 

52# to indicate whether default was defined for search. 

53# And None is a valid default value. 

54_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__" 

55 

56 

57class BpsFormatter(string.Formatter): 

58 """String formatter class that allows BPS config search options.""" 

59 

60 def get_field(self, field_name, args, kwargs): 

61 _, val = args[0].search(field_name, opt=args[1]) 

62 return val, field_name 

63 

64 def get_value(self, key, args, kwargs): 

65 _, val = args[0].search(key, opt=args[1]) 

66 return val 

67 

68 

69class BpsConfig(Config): 

70 """Contains the configuration for a BPS submission. 

71 

72 Parameters 

73 ---------- 

74 other : `str`, `dict`, `~lsst.daf.butler.Config`, `BpsConfig` 

75 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

76 to copy. 

77 search_order : `list` [`str`], optional 

78 Root section names in the order in which they should be searched. 

79 """ 

80 

81 def __init__(self, other, search_order=None): 

82 # In BPS config, the same setting can be defined multiple times in 

83 # different sections. The sections are search in a pre-defined 

84 # order. Hence, a value which is found first effectively overrides 

85 # values in later sections, if any. To achieve this goal, 

86 # the special methods __getitem__ and __contains__ were redefined to 

87 # use a custom search function internally. For this reason we can't 

88 # use super().__init__(other) as the super class defines its own 

89 # __getitem__ which is utilized during the initialization process ( 

90 # e.g. in expressions like self[<key>]). However, this function will 

91 # be overridden by the one defined here, in the subclass. Instead 

92 # we just initialize internal data structures and populate them 

93 # using the inherited update() method which does not rely on super 

94 # class __getitem__ method. 

95 super().__init__() 

96 

97 if isinstance(other, str): 

98 # First load default config from ctrl_bps, then override with 

99 # user config. 

100 tmp_config = Config(BPS_DEFAULTS) 

101 user_config = Config(other) 

102 tmp_config.update(user_config) 

103 other = tmp_config 

104 if search_order is None: 

105 search_order = BPS_SEARCH_ORDER 

106 

107 try: 

108 config = Config(other) 

109 except RuntimeError: 

110 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}") 

111 self.update(config) 

112 

113 if isinstance(other, BpsConfig): 

114 self.search_order = copy.deepcopy(other.search_order) 

115 self.formatter = copy.deepcopy(other.formatter) 

116 else: 

117 if search_order is None: 

118 search_order = [] 

119 self.search_order = search_order 

120 self.formatter = BpsFormatter() 

121 

122 # Make sure search sections exist 

123 for key in self.search_order: 

124 if not Config.__contains__(self, key): 

125 self[key] = {} 

126 

127 def copy(self): 

128 """Make a copy of config. 

129 

130 Returns 

131 ------- 

132 copy : `lsst.ctrl.bps.BpsConfig` 

133 A duplicate of itself. 

134 """ 

135 return BpsConfig(self) 

136 

137 def get(self, key, default=""): 

138 """Return the value for key if key is in the config, else default. 

139 

140 If default is not given, it defaults to an empty string. 

141 

142 Parameters 

143 ---------- 

144 key : `str` 

145 Key to look for in config. 

146 default : Any, optional 

147 Default value to return if the key is not in the config. 

148 

149 Returns 

150 ------- 

151 val : Any 

152 Value from config if found, default otherwise. 

153 

154 Notes 

155 ----- 

156 The provided default value (an empty string) was chosen to maintain 

157 the internal consistency with other methods of the class. 

158 """ 

159 _, val = self.search(key, opt={"default": default}) 

160 return val 

161 

162 def __getitem__(self, name): 

163 """Return the value from the config for the given name. 

164 

165 Parameters 

166 ---------- 

167 name : `str` 

168 Key to look for in config 

169 

170 Returns 

171 ------- 

172 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

173 Value from config if found. 

174 """ 

175 _, val = self.search(name, {}) 

176 

177 return val 

178 

179 def __contains__(self, name): 

180 """Check whether name is in config. 

181 

182 Parameters 

183 ---------- 

184 name : `str` 

185 Key to look for in config. 

186 

187 Returns 

188 ------- 

189 found : `bool` 

190 Whether name was in config or not. 

191 """ 

192 found, _ = self.search(name, {}) 

193 return found 

194 

195 def search(self, key, opt=None): 

196 """Search for key using given opt following hierarchy rules. 

197 

198 Search hierarchy rules: current values, a given search object, and 

199 search order of config sections. 

200 

201 Parameters 

202 ---------- 

203 key : `str` 

204 Key to look for in config. 

205 opt : `dict` [`str`, `Any`], optional 

206 Options dictionary to use while searching. All are optional. 

207 

208 ``"curvals"`` 

209 Means to pass in values for search order key 

210 (curr_<sectname>) or variable replacements. 

211 (`dict`, optional) 

212 ``"default"`` 

213 Value to return if not found. (`Any`, optional) 

214 ``"replaceEnvVars"`` 

215 If search result is string, whether to replace environment 

216 variables inside it with special placeholder (<ENV:name>). 

217 By default set to False. (`bool`) 

218 ``"expandEnvVars"`` 

219 If search result is string, whether to replace environment 

220 variables inside it with current environment value. 

221 By default set to False. (`bool`) 

222 ``"replaceVars"`` 

223 If search result is string, whether to replace variables 

224 inside it. By default set to True. (`bool`) 

225 ``"required"`` 

226 If replacing variables, whether to raise exception if 

227 variable is undefined. By default set to False. (`bool`) 

228 

229 Returns 

230 ------- 

231 found : `bool` 

232 Whether name was in config or not. 

233 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

234 Value from config if found. 

235 """ 

236 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

237 

238 if opt is None: 

239 opt = {} 

240 

241 found = False 

242 value = "" 

243 

244 # start with stored current values 

245 curvals = None 

246 if Config.__contains__(self, "current"): 

247 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

248 else: 

249 curvals = {} 

250 

251 # override with current values passed into function if given 

252 if "curvals" in opt: 

253 for ckey, cval in list(opt["curvals"].items()): 

254 _LOG.debug("using specified curval %s = %s", ckey, cval) 

255 curvals[ckey] = cval 

256 

257 _LOG.debug("curvals = %s", curvals) 

258 

259 # There's a problem with the searchobj being a BpsConfig 

260 # and its handling of __getitem__. Until that part of 

261 # BpsConfig is rewritten, force the searchobj to a Config. 

262 if "searchobj" in opt: 

263 opt["searchobj"] = Config(opt["searchobj"]) 

264 

265 if key in curvals: 

266 _LOG.debug("found %s in curvals", key) 

267 found = True 

268 value = curvals[key] 

269 elif "searchobj" in opt and key in opt["searchobj"]: 

270 found = True 

271 value = opt["searchobj"][key] 

272 else: 

273 for sect in self.search_order: 

274 if Config.__contains__(self, sect): 

275 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

276 search_sect = Config.__getitem__(self, sect) 

277 if "curr_" + sect in curvals: 

278 currkey = curvals["curr_" + sect] 

279 _LOG.debug("currkey for section %s = %s", sect, currkey) 

280 if Config.__contains__(search_sect, currkey): 

281 search_sect = Config.__getitem__(search_sect, currkey) 

282 

283 _LOG.debug("%s %s", key, search_sect) 

284 if Config.__contains__(search_sect, key): 

285 found = True 

286 value = Config.__getitem__(search_sect, key) 

287 break 

288 else: 

289 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

290 

291 # lastly check root values 

292 if not found: 

293 _LOG.debug("Searching root section for key '%s'", key) 

294 if Config.__contains__(self, key): 

295 found = True 

296 value = Config.__getitem__(self, key) 

297 _LOG.debug("root value='%s'", value) 

298 

299 if not found and "default" in opt: 

300 value = opt["default"] 

301 found = True # ???? 

302 

303 if not found and opt.get("required", False): 

304 print(f"\n\nError: search for {key} failed") 

305 print("\tcurrent = ", self.get("current")) 

306 print("\topt = ", opt) 

307 print("\tcurvals = ", curvals) 

308 print("\n\n") 

309 raise KeyError(f"Error: Search failed {key}") 

310 

311 _LOG.debug("found=%s, value=%s", found, value) 

312 

313 _LOG.debug("opt=%s %s", opt, type(opt)) 

314 if found and isinstance(value, str): 

315 if opt.get("expandEnvVars", True): 

316 _LOG.debug("before format=%s", value) 

317 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

318 value = expandvars(value) 

319 elif opt.get("replaceEnvVars", False): 

320 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

321 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

322 

323 if opt.get("replaceVars", True): 

324 # default only applies to original search key 

325 # Instead of doing deep copies of opt (especially with 

326 # the recursive calls), temporarily remove default value 

327 # and put it back. 

328 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE) 

329 

330 # Temporarily replace any env vars so formatter doesn't try to 

331 # replace them. 

332 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

333 

334 value = self.formatter.format(value, self, opt) 

335 

336 # Replace any temporary env place holders. 

337 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

338 

339 # if default was originally in opt 

340 if default != _NO_SEARCH_DEFAULT_VALUE: 

341 opt["default"] = default 

342 

343 _LOG.debug("after format=%s", value) 

344 

345 if found and isinstance(value, Config): 

346 value = BpsConfig(value) 

347 

348 return found, value