Coverage for python/lsst/ctrl/bps/bps_config.py: 12%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

151 statements  

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Configuration class that adds order to searching sections for value, 

23expands environment variables and other config variables. 

24""" 

25 

26__all__ = ["BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

27 

28 

29from os.path import expandvars 

30import logging 

31import copy 

32import string 

33import re 

34from importlib.resources import path as resources_path 

35import inflection 

36 

37from lsst.daf.butler.core.config import Config 

38 

39from . import etc 

40 

41_LOG = logging.getLogger(__name__) 

42 

43BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "bps_defined"] 

44 

45# Need a string that won't be a valid default value 

46# to indicate whether default was defined for search. 

47# And None is a valid default value. 

48_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__" 

49 

50 

51class BpsFormatter(string.Formatter): 

52 """String formatter class that allows BPS config search options. 

53 """ 

54 def get_field(self, field_name, args, kwargs): 

55 _, val = args[0].search(field_name, opt=args[1]) 

56 return val, field_name 

57 

58 def get_value(self, key, args, kwargs): 

59 _, val = args[0].search(key, opt=args[1]) 

60 return val 

61 

62 

63class BpsConfig(Config): 

64 """Contains the configuration for a BPS submission. 

65 

66 Parameters 

67 ---------- 

68 other : `str`, `dict`, `Config`, `BpsConfig` 

69 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

70 to copy. 

71 search_order : `list` [`str`], optional 

72 Root section names in the order in which they should be searched. 

73 """ 

74 def __init__(self, other, search_order=None): 

75 # In BPS config, the same setting can be defined multiple times in 

76 # different sections. The sections are search in a pre-defined 

77 # order. Hence, a value which is found first effectively overrides 

78 # values in later sections, if any. To achieve this goal, 

79 # the special methods __getitem__ and __contains__ were redefined to 

80 # use a custom search function internally. For this reason we can't 

81 # use super().__init__(other) as the super class defines its own 

82 # __getitem__ which is utilized during the initialization process ( 

83 # e.g. in expressions like self[<key>]). However, this function will 

84 # be overridden by the one defined here, in the subclass. Instead 

85 # we just initialize internal data structures and populate them 

86 # using the inherited update() method which does not rely on super 

87 # class __getitem__ method. 

88 super().__init__() 

89 

90 if isinstance(other, str): 

91 # First load default config from ctrl_bps, then override with 

92 # user config. 

93 with resources_path(etc, "bps_defaults.yaml") as bps_defaults: 

94 tmp_config = Config(str(bps_defaults)) 

95 user_config = Config(other) 

96 tmp_config.update(user_config) 

97 other = tmp_config 

98 if search_order is None: 

99 search_order = BPS_SEARCH_ORDER 

100 

101 try: 

102 config = Config(other) 

103 except RuntimeError: 

104 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}") 

105 self.update(config) 

106 

107 if isinstance(other, BpsConfig): 

108 self.search_order = copy.deepcopy(other.search_order) 

109 self.formatter = copy.deepcopy(other.formatter) 

110 else: 

111 if search_order is None: 

112 search_order = [] 

113 self.search_order = search_order 

114 self.formatter = BpsFormatter() 

115 

116 # Make sure search sections exist 

117 for key in self.search_order: 

118 if not Config.__contains__(self, key): 

119 self[key] = {} 

120 

121 def copy(self): 

122 """Make a copy of config. 

123 

124 Returns 

125 ------- 

126 copy : `lsst.ctrl.bps.BpsConfig` 

127 A duplicate of itself. 

128 """ 

129 return BpsConfig(self) 

130 

131 def __getitem__(self, name): 

132 """Return the value from the config for the given name. 

133 

134 Parameters 

135 ---------- 

136 name : `str` 

137 Key to look for in config 

138 

139 Returns 

140 ------- 

141 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

142 Value from config if found. 

143 """ 

144 _, val = self.search(name, {}) 

145 

146 return val 

147 

148 def __contains__(self, name): 

149 """Check whether name is in config. 

150 

151 Parameters 

152 ---------- 

153 name : `str` 

154 Key to look for in config. 

155 

156 Returns 

157 ------- 

158 found : `bool` 

159 Whether name was in config or not. 

160 """ 

161 found, _ = self.search(name, {}) 

162 return found 

163 

164 @staticmethod 

165 def _search_casing(sect, key): 

166 # Until have more robust handling of key casing at config creation 

167 # time, try checking here for different key casing. 

168 found = False 

169 value = "" 

170 

171 _LOG.debug("_search_casing: sect=%s key=%s", sect, key) 

172 if Config.__contains__(sect, key): 

173 found = True 

174 value = Config.__getitem__(sect, key) 

175 elif '_' in key: 

176 newkey = inflection.camelize(key, False) 

177 _LOG.debug("_search_casing: newkey=%s", newkey) 

178 if Config.__contains__(sect, newkey): 

179 found = True 

180 value = Config.__getitem__(sect, newkey) 

181 else: # try converting camel to snake 

182 newkey = inflection.underscore(key) 

183 _LOG.debug("_search_casing: newkey=%s", newkey) 

184 if Config.__contains__(sect, newkey): 

185 found = True 

186 value = Config.__getitem__(sect, newkey) 

187 else: # Try all lower case 

188 newkey = key.lower() 

189 _LOG.debug("_search_casing: newkey=%s", newkey) 

190 if Config.__contains__(sect, newkey): 

191 found = True 

192 value = Config.__getitem__(sect, newkey) 

193 

194 return found, value 

195 

196 def search(self, key, opt=None): 

197 """Search for key using given opt following hierarchy rules. 

198 

199 Search hierarchy rules: current values, a given search object, and 

200 search order of config sections. 

201 

202 Parameters 

203 ---------- 

204 key : `str` 

205 Key to look for in config. 

206 opt : `dict` [`str`, `Any`], optional 

207 Options dictionary to use while searching. All are optional. 

208 

209 ``"curvals"`` 

210 Means to pass in values for search order key 

211 (curr_<sectname>) or variable replacements. 

212 (`dict`, optional) 

213 ``"default"`` 

214 Value to return if not found. (`Any`, optional) 

215 ``"replaceEnvVars"`` 

216 If search result is string, whether to replace environment 

217 variables inside it with special placeholder (<ENV:name>). 

218 By default set to False. (`bool`) 

219 ``"expandEnvVars"`` 

220 If search result is string, whether to replace environment 

221 variables inside it with current environment value. 

222 By default set to False. (`bool`) 

223 ``"replaceVars"`` 

224 If search result is string, whether to replace variables 

225 inside it. By default set to True. (`bool`) 

226 ``"required"`` 

227 If replacing variables, whether to raise exception if 

228 variable is undefined. By default set to False. (`bool`) 

229 

230 Returns 

231 ------- 

232 found : `bool` 

233 Whether name was in config or not. 

234 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

235 Value from config if found. 

236 """ 

237 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

238 

239 if opt is None: 

240 opt = {} 

241 

242 found = False 

243 value = "" 

244 

245 # start with stored current values 

246 curvals = None 

247 if Config.__contains__(self, "current"): 

248 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

249 else: 

250 curvals = {} 

251 

252 # override with current values passed into function if given 

253 if "curvals" in opt: 

254 for ckey, cval in list(opt["curvals"].items()): 

255 _LOG.debug("using specified curval %s = %s", ckey, cval) 

256 curvals[ckey] = cval 

257 

258 _LOG.debug("curvals = %s", curvals) 

259 

260 # There's a problem with the searchobj being a BpsConfig 

261 # and its handling of __getitem__. Until that part of 

262 # BpsConfig is rewritten, force the searchobj to a Config. 

263 if "searchobj" in opt: 

264 opt["searchobj"] = Config(opt["searchobj"]) 

265 

266 if key in curvals: 

267 _LOG.debug("found %s in curvals", key) 

268 found = True 

269 value = curvals[key] 

270 elif "searchobj" in opt and key in opt["searchobj"]: 

271 found = True 

272 value = opt["searchobj"][key] 

273 else: 

274 for sect in self.search_order: 

275 if Config.__contains__(self, sect): 

276 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

277 search_sect = Config.__getitem__(self, sect) 

278 if "curr_" + sect in curvals: 

279 currkey = curvals["curr_" + sect] 

280 _LOG.debug("currkey for section %s = %s", sect, currkey) 

281 if Config.__contains__(search_sect, currkey): 

282 search_sect = Config.__getitem__(search_sect, currkey) 

283 

284 found, value = self._search_casing(search_sect, key) 

285 if found: 

286 break 

287 

288 # lastly check root values 

289 if not found: 

290 _LOG.debug("Searching root section for key '%s'", key) 

291 found, value = self._search_casing(self, key) 

292 _LOG.debug(" root found=%s, value='%s'", found, value) 

293 

294 if not found and "default" in opt: 

295 value = opt["default"] 

296 found = True # ???? 

297 

298 if not found and opt.get("required", False): 

299 print(f"\n\nError: search for {key} failed") 

300 print("\tcurrent = ", self.get("current")) 

301 print("\topt = ", opt) 

302 print("\tcurvals = ", curvals) 

303 print("\n\n") 

304 raise KeyError(f"Error: Search failed {key}") 

305 

306 _LOG.debug("found=%s, value=%s", found, value) 

307 

308 _LOG.debug("opt=%s %s", opt, type(opt)) 

309 if found and isinstance(value, str): 

310 if opt.get("expandEnvVars", True): 

311 _LOG.debug("before format=%s", value) 

312 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

313 value = expandvars(value) 

314 elif opt.get("replaceEnvVars", False): 

315 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

316 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

317 

318 if opt.get("replaceVars", True): 

319 # default only applies to original search key 

320 # Instead of doing deep copies of opt (especially with 

321 # the recursive calls), temporarily remove default value 

322 # and put it back. 

323 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE) 

324 

325 # Temporarily replace any env vars so formatter doesn't try to 

326 # replace them. 

327 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

328 

329 value = self.formatter.format(value, self, opt) 

330 

331 # Replace any temporary env place holders. 

332 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

333 

334 # if default was originally in opt 

335 if default != _NO_SEARCH_DEFAULT_VALUE: 

336 opt["default"] = default 

337 

338 _LOG.debug("after format=%s", value) 

339 

340 if found and isinstance(value, Config): 

341 value = BpsConfig(value) 

342 

343 return found, value