Coverage for python/lsst/ctrl/bps/bps_config.py: 14%

134 statements  

« prev     ^ index     » next       coverage.py v7.2.5, created at 2023-05-09 02:20 -0700

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Configuration class that adds order to searching sections for value, 

23expands environment variables and other config variables. 

24""" 

25 

26__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

27 

28 

29import copy 

30import logging 

31import re 

32import string 

33from importlib import resources 

34from os.path import expandvars 

35 

36from lsst.daf.butler.core.config import Config 

37 

38from . import etc 

39 

40_LOG = logging.getLogger(__name__) 

41 

42# Using lsst.daf.butler.Config to resolve possible includes. 

43with resources.path(etc, "bps_defaults.yaml") as path: 

44 BPS_DEFAULTS = Config(str(path)).toDict() 

45 

46BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "cloud", "bps_defined"] 

47 

48# Need a string that won't be a valid default value 

49# to indicate whether default was defined for search. 

50# And None is a valid default value. 

51_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__" 

52 

53 

54class BpsFormatter(string.Formatter): 

55 """String formatter class that allows BPS config search options.""" 

56 

57 def get_field(self, field_name, args, kwargs): 

58 _, val = args[0].search(field_name, opt=args[1]) 

59 return val, field_name 

60 

61 def get_value(self, key, args, kwargs): 

62 _, val = args[0].search(key, opt=args[1]) 

63 return val 

64 

65 

66class BpsConfig(Config): 

67 """Contains the configuration for a BPS submission. 

68 

69 Parameters 

70 ---------- 

71 other : `str`, `dict`, `Config`, `BpsConfig` 

72 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

73 to copy. 

74 search_order : `list` [`str`], optional 

75 Root section names in the order in which they should be searched. 

76 """ 

77 

78 def __init__(self, other, search_order=None): 

79 # In BPS config, the same setting can be defined multiple times in 

80 # different sections. The sections are search in a pre-defined 

81 # order. Hence, a value which is found first effectively overrides 

82 # values in later sections, if any. To achieve this goal, 

83 # the special methods __getitem__ and __contains__ were redefined to 

84 # use a custom search function internally. For this reason we can't 

85 # use super().__init__(other) as the super class defines its own 

86 # __getitem__ which is utilized during the initialization process ( 

87 # e.g. in expressions like self[<key>]). However, this function will 

88 # be overridden by the one defined here, in the subclass. Instead 

89 # we just initialize internal data structures and populate them 

90 # using the inherited update() method which does not rely on super 

91 # class __getitem__ method. 

92 super().__init__() 

93 

94 if isinstance(other, str): 

95 # First load default config from ctrl_bps, then override with 

96 # user config. 

97 tmp_config = Config(BPS_DEFAULTS) 

98 user_config = Config(other) 

99 tmp_config.update(user_config) 

100 other = tmp_config 

101 if search_order is None: 

102 search_order = BPS_SEARCH_ORDER 

103 

104 try: 

105 config = Config(other) 

106 except RuntimeError: 

107 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}") 

108 self.update(config) 

109 

110 if isinstance(other, BpsConfig): 

111 self.search_order = copy.deepcopy(other.search_order) 

112 self.formatter = copy.deepcopy(other.formatter) 

113 else: 

114 if search_order is None: 

115 search_order = [] 

116 self.search_order = search_order 

117 self.formatter = BpsFormatter() 

118 

119 # Make sure search sections exist 

120 for key in self.search_order: 

121 if not Config.__contains__(self, key): 

122 self[key] = {} 

123 

124 def copy(self): 

125 """Make a copy of config. 

126 

127 Returns 

128 ------- 

129 copy : `lsst.ctrl.bps.BpsConfig` 

130 A duplicate of itself. 

131 """ 

132 return BpsConfig(self) 

133 

134 def get(self, key, default=""): 

135 """Return the value for key if key is in the config, else default. 

136 

137 If default is not given, it defaults to an empty string. 

138 

139 Parameters 

140 ---------- 

141 key : `str` 

142 Key to look for in config. 

143 default : Any, optional 

144 Default value to return if the key is not in the config. 

145 

146 Returns 

147 ------- 

148 val : Any 

149 Value from config if found, default otherwise. 

150 

151 Notes 

152 ----- 

153 The provided default value (an empty string) was chosen to maintain 

154 the internal consistency with other methods of the class. 

155 """ 

156 _, val = self.search(key, opt={"default": default}) 

157 return val 

158 

159 def __getitem__(self, name): 

160 """Return the value from the config for the given name. 

161 

162 Parameters 

163 ---------- 

164 name : `str` 

165 Key to look for in config 

166 

167 Returns 

168 ------- 

169 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

170 Value from config if found. 

171 """ 

172 _, val = self.search(name, {}) 

173 

174 return val 

175 

176 def __contains__(self, name): 

177 """Check whether name is in config. 

178 

179 Parameters 

180 ---------- 

181 name : `str` 

182 Key to look for in config. 

183 

184 Returns 

185 ------- 

186 found : `bool` 

187 Whether name was in config or not. 

188 """ 

189 found, _ = self.search(name, {}) 

190 return found 

191 

192 def search(self, key, opt=None): 

193 """Search for key using given opt following hierarchy rules. 

194 

195 Search hierarchy rules: current values, a given search object, and 

196 search order of config sections. 

197 

198 Parameters 

199 ---------- 

200 key : `str` 

201 Key to look for in config. 

202 opt : `dict` [`str`, `Any`], optional 

203 Options dictionary to use while searching. All are optional. 

204 

205 ``"curvals"`` 

206 Means to pass in values for search order key 

207 (curr_<sectname>) or variable replacements. 

208 (`dict`, optional) 

209 ``"default"`` 

210 Value to return if not found. (`Any`, optional) 

211 ``"replaceEnvVars"`` 

212 If search result is string, whether to replace environment 

213 variables inside it with special placeholder (<ENV:name>). 

214 By default set to False. (`bool`) 

215 ``"expandEnvVars"`` 

216 If search result is string, whether to replace environment 

217 variables inside it with current environment value. 

218 By default set to False. (`bool`) 

219 ``"replaceVars"`` 

220 If search result is string, whether to replace variables 

221 inside it. By default set to True. (`bool`) 

222 ``"required"`` 

223 If replacing variables, whether to raise exception if 

224 variable is undefined. By default set to False. (`bool`) 

225 

226 Returns 

227 ------- 

228 found : `bool` 

229 Whether name was in config or not. 

230 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

231 Value from config if found. 

232 """ 

233 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

234 

235 if opt is None: 

236 opt = {} 

237 

238 found = False 

239 value = "" 

240 

241 # start with stored current values 

242 curvals = None 

243 if Config.__contains__(self, "current"): 

244 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

245 else: 

246 curvals = {} 

247 

248 # override with current values passed into function if given 

249 if "curvals" in opt: 

250 for ckey, cval in list(opt["curvals"].items()): 

251 _LOG.debug("using specified curval %s = %s", ckey, cval) 

252 curvals[ckey] = cval 

253 

254 _LOG.debug("curvals = %s", curvals) 

255 

256 # There's a problem with the searchobj being a BpsConfig 

257 # and its handling of __getitem__. Until that part of 

258 # BpsConfig is rewritten, force the searchobj to a Config. 

259 if "searchobj" in opt: 

260 opt["searchobj"] = Config(opt["searchobj"]) 

261 

262 if key in curvals: 

263 _LOG.debug("found %s in curvals", key) 

264 found = True 

265 value = curvals[key] 

266 elif "searchobj" in opt and key in opt["searchobj"]: 

267 found = True 

268 value = opt["searchobj"][key] 

269 else: 

270 for sect in self.search_order: 

271 if Config.__contains__(self, sect): 

272 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

273 search_sect = Config.__getitem__(self, sect) 

274 if "curr_" + sect in curvals: 

275 currkey = curvals["curr_" + sect] 

276 _LOG.debug("currkey for section %s = %s", sect, currkey) 

277 if Config.__contains__(search_sect, currkey): 

278 search_sect = Config.__getitem__(search_sect, currkey) 

279 

280 _LOG.debug("%s %s", key, search_sect) 

281 if Config.__contains__(search_sect, key): 

282 found = True 

283 value = Config.__getitem__(search_sect, key) 

284 break 

285 else: 

286 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

287 

288 # lastly check root values 

289 if not found: 

290 _LOG.debug("Searching root section for key '%s'", key) 

291 if Config.__contains__(self, key): 

292 found = True 

293 value = Config.__getitem__(self, key) 

294 _LOG.debug("root value='%s'", value) 

295 

296 if not found and "default" in opt: 

297 value = opt["default"] 

298 found = True # ???? 

299 

300 if not found and opt.get("required", False): 

301 print(f"\n\nError: search for {key} failed") 

302 print("\tcurrent = ", self.get("current")) 

303 print("\topt = ", opt) 

304 print("\tcurvals = ", curvals) 

305 print("\n\n") 

306 raise KeyError(f"Error: Search failed {key}") 

307 

308 _LOG.debug("found=%s, value=%s", found, value) 

309 

310 _LOG.debug("opt=%s %s", opt, type(opt)) 

311 if found and isinstance(value, str): 

312 if opt.get("expandEnvVars", True): 

313 _LOG.debug("before format=%s", value) 

314 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

315 value = expandvars(value) 

316 elif opt.get("replaceEnvVars", False): 

317 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

318 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

319 

320 if opt.get("replaceVars", True): 

321 # default only applies to original search key 

322 # Instead of doing deep copies of opt (especially with 

323 # the recursive calls), temporarily remove default value 

324 # and put it back. 

325 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE) 

326 

327 # Temporarily replace any env vars so formatter doesn't try to 

328 # replace them. 

329 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

330 

331 value = self.formatter.format(value, self, opt) 

332 

333 # Replace any temporary env place holders. 

334 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

335 

336 # if default was originally in opt 

337 if default != _NO_SEARCH_DEFAULT_VALUE: 

338 opt["default"] = default 

339 

340 _LOG.debug("after format=%s", value) 

341 

342 if found and isinstance(value, Config): 

343 value = BpsConfig(value) 

344 

345 return found, value