Coverage for python/lsst/ctrl/bps/bps_config.py: 14%

131 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-01 23:59 -0700

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Configuration class that adds order to searching sections for value, 

23expands environment variables and other config variables. 

24""" 

25 

26__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

27 

28 

29import copy 

30import logging 

31import re 

32import string 

33from importlib import resources 

34from os.path import expandvars 

35 

36from lsst.daf.butler.core.config import Config 

37 

38from . import etc 

39 

40_LOG = logging.getLogger(__name__) 

41 

42# Using lsst.daf.butler.Config to resolve possible includes. 

43with resources.path(etc, "bps_defaults.yaml") as path: 

44 BPS_DEFAULTS = Config(str(path)).toDict() 

45 

46BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "cloud", "bps_defined"] 

47 

48# Need a string that won't be a valid default value 

49# to indicate whether default was defined for search. 

50# And None is a valid default value. 

51_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__" 

52 

53 

54class BpsFormatter(string.Formatter): 

55 """String formatter class that allows BPS config search options.""" 

56 

57 def get_field(self, field_name, args, kwargs): 

58 _, val = args[0].search(field_name, opt=args[1]) 

59 return val, field_name 

60 

61 def get_value(self, key, args, kwargs): 

62 _, val = args[0].search(key, opt=args[1]) 

63 return val 

64 

65 

66class BpsConfig(Config): 

67 """Contains the configuration for a BPS submission. 

68 

69 Parameters 

70 ---------- 

71 other : `str`, `dict`, `Config`, `BpsConfig` 

72 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

73 to copy. 

74 search_order : `list` [`str`], optional 

75 Root section names in the order in which they should be searched. 

76 """ 

77 

78 def __init__(self, other, search_order=None): 

79 # In BPS config, the same setting can be defined multiple times in 

80 # different sections. The sections are search in a pre-defined 

81 # order. Hence, a value which is found first effectively overrides 

82 # values in later sections, if any. To achieve this goal, 

83 # the special methods __getitem__ and __contains__ were redefined to 

84 # use a custom search function internally. For this reason we can't 

85 # use super().__init__(other) as the super class defines its own 

86 # __getitem__ which is utilized during the initialization process ( 

87 # e.g. in expressions like self[<key>]). However, this function will 

88 # be overridden by the one defined here, in the subclass. Instead 

89 # we just initialize internal data structures and populate them 

90 # using the inherited update() method which does not rely on super 

91 # class __getitem__ method. 

92 super().__init__() 

93 

94 if isinstance(other, str): 

95 # First load default config from ctrl_bps, then override with 

96 # user config. 

97 tmp_config = Config(BPS_DEFAULTS) 

98 user_config = Config(other) 

99 tmp_config.update(user_config) 

100 other = tmp_config 

101 if search_order is None: 

102 search_order = BPS_SEARCH_ORDER 

103 

104 try: 

105 config = Config(other) 

106 except RuntimeError: 

107 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}") 

108 self.update(config) 

109 

110 if isinstance(other, BpsConfig): 

111 self.search_order = copy.deepcopy(other.search_order) 

112 self.formatter = copy.deepcopy(other.formatter) 

113 else: 

114 if search_order is None: 

115 search_order = [] 

116 self.search_order = search_order 

117 self.formatter = BpsFormatter() 

118 

119 # Make sure search sections exist 

120 for key in self.search_order: 

121 if not Config.__contains__(self, key): 

122 self[key] = {} 

123 

124 def copy(self): 

125 """Make a copy of config. 

126 

127 Returns 

128 ------- 

129 copy : `lsst.ctrl.bps.BpsConfig` 

130 A duplicate of itself. 

131 """ 

132 return BpsConfig(self) 

133 

134 def __getitem__(self, name): 

135 """Return the value from the config for the given name. 

136 

137 Parameters 

138 ---------- 

139 name : `str` 

140 Key to look for in config 

141 

142 Returns 

143 ------- 

144 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

145 Value from config if found. 

146 """ 

147 _, val = self.search(name, {}) 

148 

149 return val 

150 

151 def __contains__(self, name): 

152 """Check whether name is in config. 

153 

154 Parameters 

155 ---------- 

156 name : `str` 

157 Key to look for in config. 

158 

159 Returns 

160 ------- 

161 found : `bool` 

162 Whether name was in config or not. 

163 """ 

164 found, _ = self.search(name, {}) 

165 return found 

166 

167 def search(self, key, opt=None): 

168 """Search for key using given opt following hierarchy rules. 

169 

170 Search hierarchy rules: current values, a given search object, and 

171 search order of config sections. 

172 

173 Parameters 

174 ---------- 

175 key : `str` 

176 Key to look for in config. 

177 opt : `dict` [`str`, `Any`], optional 

178 Options dictionary to use while searching. All are optional. 

179 

180 ``"curvals"`` 

181 Means to pass in values for search order key 

182 (curr_<sectname>) or variable replacements. 

183 (`dict`, optional) 

184 ``"default"`` 

185 Value to return if not found. (`Any`, optional) 

186 ``"replaceEnvVars"`` 

187 If search result is string, whether to replace environment 

188 variables inside it with special placeholder (<ENV:name>). 

189 By default set to False. (`bool`) 

190 ``"expandEnvVars"`` 

191 If search result is string, whether to replace environment 

192 variables inside it with current environment value. 

193 By default set to False. (`bool`) 

194 ``"replaceVars"`` 

195 If search result is string, whether to replace variables 

196 inside it. By default set to True. (`bool`) 

197 ``"required"`` 

198 If replacing variables, whether to raise exception if 

199 variable is undefined. By default set to False. (`bool`) 

200 

201 Returns 

202 ------- 

203 found : `bool` 

204 Whether name was in config or not. 

205 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

206 Value from config if found. 

207 """ 

208 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

209 

210 if opt is None: 

211 opt = {} 

212 

213 found = False 

214 value = "" 

215 

216 # start with stored current values 

217 curvals = None 

218 if Config.__contains__(self, "current"): 

219 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

220 else: 

221 curvals = {} 

222 

223 # override with current values passed into function if given 

224 if "curvals" in opt: 

225 for ckey, cval in list(opt["curvals"].items()): 

226 _LOG.debug("using specified curval %s = %s", ckey, cval) 

227 curvals[ckey] = cval 

228 

229 _LOG.debug("curvals = %s", curvals) 

230 

231 # There's a problem with the searchobj being a BpsConfig 

232 # and its handling of __getitem__. Until that part of 

233 # BpsConfig is rewritten, force the searchobj to a Config. 

234 if "searchobj" in opt: 

235 opt["searchobj"] = Config(opt["searchobj"]) 

236 

237 if key in curvals: 

238 _LOG.debug("found %s in curvals", key) 

239 found = True 

240 value = curvals[key] 

241 elif "searchobj" in opt and key in opt["searchobj"]: 

242 found = True 

243 value = opt["searchobj"][key] 

244 else: 

245 for sect in self.search_order: 

246 if Config.__contains__(self, sect): 

247 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

248 search_sect = Config.__getitem__(self, sect) 

249 if "curr_" + sect in curvals: 

250 currkey = curvals["curr_" + sect] 

251 _LOG.debug("currkey for section %s = %s", sect, currkey) 

252 if Config.__contains__(search_sect, currkey): 

253 search_sect = Config.__getitem__(search_sect, currkey) 

254 

255 _LOG.debug("%s %s", key, search_sect) 

256 if Config.__contains__(search_sect, key): 

257 found = True 

258 value = Config.__getitem__(search_sect, key) 

259 break 

260 else: 

261 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

262 

263 # lastly check root values 

264 if not found: 

265 _LOG.debug("Searching root section for key '%s'", key) 

266 if Config.__contains__(self, key): 

267 found = True 

268 value = Config.__getitem__(self, key) 

269 _LOG.debug("root value='%s'", value) 

270 

271 if not found and "default" in opt: 

272 value = opt["default"] 

273 found = True # ???? 

274 

275 if not found and opt.get("required", False): 

276 print(f"\n\nError: search for {key} failed") 

277 print("\tcurrent = ", self.get("current")) 

278 print("\topt = ", opt) 

279 print("\tcurvals = ", curvals) 

280 print("\n\n") 

281 raise KeyError(f"Error: Search failed {key}") 

282 

283 _LOG.debug("found=%s, value=%s", found, value) 

284 

285 _LOG.debug("opt=%s %s", opt, type(opt)) 

286 if found and isinstance(value, str): 

287 if opt.get("expandEnvVars", True): 

288 _LOG.debug("before format=%s", value) 

289 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

290 value = expandvars(value) 

291 elif opt.get("replaceEnvVars", False): 

292 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

293 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

294 

295 if opt.get("replaceVars", True): 

296 # default only applies to original search key 

297 # Instead of doing deep copies of opt (especially with 

298 # the recursive calls), temporarily remove default value 

299 # and put it back. 

300 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE) 

301 

302 # Temporarily replace any env vars so formatter doesn't try to 

303 # replace them. 

304 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

305 

306 value = self.formatter.format(value, self, opt) 

307 

308 # Replace any temporary env place holders. 

309 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

310 

311 # if default was originally in opt 

312 if default != _NO_SEARCH_DEFAULT_VALUE: 

313 opt["default"] = default 

314 

315 _LOG.debug("after format=%s", value) 

316 

317 if found and isinstance(value, Config): 

318 value = BpsConfig(value) 

319 

320 return found, value