Coverage for python/lsst/ctrl/bps/bps_config.py: 12%

132 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 10:39 +0000

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Configuration class that adds order to searching sections for value, 

23expands environment variables and other config variables. 

24""" 

25 

26__all__ = ["BPS_DEFAULTS", "BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

27 

28 

29import copy 

30import logging 

31import re 

32import string 

33from os.path import expandvars 

34 

35from lsst.daf.butler import Config 

36from lsst.resources import ResourcePath 

37 

38_LOG = logging.getLogger(__name__) 

39 

40# Using lsst.daf.butler.Config to resolve possible includes. 

41BPS_DEFAULTS = Config(ResourcePath("resource://lsst.ctrl.bps/etc/bps_defaults.yaml")).toDict() 

42 

43BPS_SEARCH_ORDER = ["bps_cmdline", "payload", "cluster", "pipetask", "site", "cloud", "bps_defined"] 

44 

45# Need a string that won't be a valid default value 

46# to indicate whether default was defined for search. 

47# And None is a valid default value. 

48_NO_SEARCH_DEFAULT_VALUE = "__NO_SEARCH_DEFAULT_VALUE__" 

49 

50 

51class BpsFormatter(string.Formatter): 

52 """String formatter class that allows BPS config search options.""" 

53 

54 def get_field(self, field_name, args, kwargs): 

55 _, val = args[0].search(field_name, opt=args[1]) 

56 return val, field_name 

57 

58 def get_value(self, key, args, kwargs): 

59 _, val = args[0].search(key, opt=args[1]) 

60 return val 

61 

62 

63class BpsConfig(Config): 

64 """Contains the configuration for a BPS submission. 

65 

66 Parameters 

67 ---------- 

68 other : `str`, `dict`, `~lsst.daf.butler.Config`, `BpsConfig` 

69 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

70 to copy. 

71 search_order : `list` [`str`], optional 

72 Root section names in the order in which they should be searched. 

73 """ 

74 

75 def __init__(self, other, search_order=None): 

76 # In BPS config, the same setting can be defined multiple times in 

77 # different sections. The sections are search in a pre-defined 

78 # order. Hence, a value which is found first effectively overrides 

79 # values in later sections, if any. To achieve this goal, 

80 # the special methods __getitem__ and __contains__ were redefined to 

81 # use a custom search function internally. For this reason we can't 

82 # use super().__init__(other) as the super class defines its own 

83 # __getitem__ which is utilized during the initialization process ( 

84 # e.g. in expressions like self[<key>]). However, this function will 

85 # be overridden by the one defined here, in the subclass. Instead 

86 # we just initialize internal data structures and populate them 

87 # using the inherited update() method which does not rely on super 

88 # class __getitem__ method. 

89 super().__init__() 

90 

91 if isinstance(other, str): 

92 # First load default config from ctrl_bps, then override with 

93 # user config. 

94 tmp_config = Config(BPS_DEFAULTS) 

95 user_config = Config(other) 

96 tmp_config.update(user_config) 

97 other = tmp_config 

98 if search_order is None: 

99 search_order = BPS_SEARCH_ORDER 

100 

101 try: 

102 config = Config(other) 

103 except RuntimeError: 

104 raise RuntimeError(f"A BpsConfig could not be loaded from other: {other}") 

105 self.update(config) 

106 

107 if isinstance(other, BpsConfig): 

108 self.search_order = copy.deepcopy(other.search_order) 

109 self.formatter = copy.deepcopy(other.formatter) 

110 else: 

111 if search_order is None: 

112 search_order = [] 

113 self.search_order = search_order 

114 self.formatter = BpsFormatter() 

115 

116 # Make sure search sections exist 

117 for key in self.search_order: 

118 if not Config.__contains__(self, key): 

119 self[key] = {} 

120 

121 def copy(self): 

122 """Make a copy of config. 

123 

124 Returns 

125 ------- 

126 copy : `lsst.ctrl.bps.BpsConfig` 

127 A duplicate of itself. 

128 """ 

129 return BpsConfig(self) 

130 

131 def get(self, key, default=""): 

132 """Return the value for key if key is in the config, else default. 

133 

134 If default is not given, it defaults to an empty string. 

135 

136 Parameters 

137 ---------- 

138 key : `str` 

139 Key to look for in config. 

140 default : Any, optional 

141 Default value to return if the key is not in the config. 

142 

143 Returns 

144 ------- 

145 val : Any 

146 Value from config if found, default otherwise. 

147 

148 Notes 

149 ----- 

150 The provided default value (an empty string) was chosen to maintain 

151 the internal consistency with other methods of the class. 

152 """ 

153 _, val = self.search(key, opt={"default": default}) 

154 return val 

155 

156 def __getitem__(self, name): 

157 """Return the value from the config for the given name. 

158 

159 Parameters 

160 ---------- 

161 name : `str` 

162 Key to look for in config 

163 

164 Returns 

165 ------- 

166 val : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

167 Value from config if found. 

168 """ 

169 _, val = self.search(name, {}) 

170 

171 return val 

172 

173 def __contains__(self, name): 

174 """Check whether name is in config. 

175 

176 Parameters 

177 ---------- 

178 name : `str` 

179 Key to look for in config. 

180 

181 Returns 

182 ------- 

183 found : `bool` 

184 Whether name was in config or not. 

185 """ 

186 found, _ = self.search(name, {}) 

187 return found 

188 

189 def search(self, key, opt=None): 

190 """Search for key using given opt following hierarchy rules. 

191 

192 Search hierarchy rules: current values, a given search object, and 

193 search order of config sections. 

194 

195 Parameters 

196 ---------- 

197 key : `str` 

198 Key to look for in config. 

199 opt : `dict` [`str`, `Any`], optional 

200 Options dictionary to use while searching. All are optional. 

201 

202 ``"curvals"`` 

203 Means to pass in values for search order key 

204 (curr_<sectname>) or variable replacements. 

205 (`dict`, optional) 

206 ``"default"`` 

207 Value to return if not found. (`Any`, optional) 

208 ``"replaceEnvVars"`` 

209 If search result is string, whether to replace environment 

210 variables inside it with special placeholder (<ENV:name>). 

211 By default set to False. (`bool`) 

212 ``"expandEnvVars"`` 

213 If search result is string, whether to replace environment 

214 variables inside it with current environment value. 

215 By default set to False. (`bool`) 

216 ``"replaceVars"`` 

217 If search result is string, whether to replace variables 

218 inside it. By default set to True. (`bool`) 

219 ``"required"`` 

220 If replacing variables, whether to raise exception if 

221 variable is undefined. By default set to False. (`bool`) 

222 

223 Returns 

224 ------- 

225 found : `bool` 

226 Whether name was in config or not. 

227 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

228 Value from config if found. 

229 """ 

230 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

231 

232 if opt is None: 

233 opt = {} 

234 

235 found = False 

236 value = "" 

237 

238 # start with stored current values 

239 curvals = None 

240 if Config.__contains__(self, "current"): 

241 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

242 else: 

243 curvals = {} 

244 

245 # override with current values passed into function if given 

246 if "curvals" in opt: 

247 for ckey, cval in list(opt["curvals"].items()): 

248 _LOG.debug("using specified curval %s = %s", ckey, cval) 

249 curvals[ckey] = cval 

250 

251 _LOG.debug("curvals = %s", curvals) 

252 

253 # There's a problem with the searchobj being a BpsConfig 

254 # and its handling of __getitem__. Until that part of 

255 # BpsConfig is rewritten, force the searchobj to a Config. 

256 if "searchobj" in opt: 

257 opt["searchobj"] = Config(opt["searchobj"]) 

258 

259 if key in curvals: 

260 _LOG.debug("found %s in curvals", key) 

261 found = True 

262 value = curvals[key] 

263 elif "searchobj" in opt and key in opt["searchobj"]: 

264 found = True 

265 value = opt["searchobj"][key] 

266 else: 

267 for sect in self.search_order: 

268 if Config.__contains__(self, sect): 

269 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

270 search_sect = Config.__getitem__(self, sect) 

271 if "curr_" + sect in curvals: 

272 currkey = curvals["curr_" + sect] 

273 _LOG.debug("currkey for section %s = %s", sect, currkey) 

274 if Config.__contains__(search_sect, currkey): 

275 search_sect = Config.__getitem__(search_sect, currkey) 

276 

277 _LOG.debug("%s %s", key, search_sect) 

278 if Config.__contains__(search_sect, key): 

279 found = True 

280 value = Config.__getitem__(search_sect, key) 

281 break 

282 else: 

283 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

284 

285 # lastly check root values 

286 if not found: 

287 _LOG.debug("Searching root section for key '%s'", key) 

288 if Config.__contains__(self, key): 

289 found = True 

290 value = Config.__getitem__(self, key) 

291 _LOG.debug("root value='%s'", value) 

292 

293 if not found and "default" in opt: 

294 value = opt["default"] 

295 found = True # ???? 

296 

297 if not found and opt.get("required", False): 

298 print(f"\n\nError: search for {key} failed") 

299 print("\tcurrent = ", self.get("current")) 

300 print("\topt = ", opt) 

301 print("\tcurvals = ", curvals) 

302 print("\n\n") 

303 raise KeyError(f"Error: Search failed {key}") 

304 

305 _LOG.debug("found=%s, value=%s", found, value) 

306 

307 _LOG.debug("opt=%s %s", opt, type(opt)) 

308 if found and isinstance(value, str): 

309 if opt.get("expandEnvVars", True): 

310 _LOG.debug("before format=%s", value) 

311 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

312 value = expandvars(value) 

313 elif opt.get("replaceEnvVars", False): 

314 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

315 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

316 

317 if opt.get("replaceVars", True): 

318 # default only applies to original search key 

319 # Instead of doing deep copies of opt (especially with 

320 # the recursive calls), temporarily remove default value 

321 # and put it back. 

322 default = opt.pop("default", _NO_SEARCH_DEFAULT_VALUE) 

323 

324 # Temporarily replace any env vars so formatter doesn't try to 

325 # replace them. 

326 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

327 

328 value = self.formatter.format(value, self, opt) 

329 

330 # Replace any temporary env place holders. 

331 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

332 

333 # if default was originally in opt 

334 if default != _NO_SEARCH_DEFAULT_VALUE: 

335 opt["default"] = default 

336 

337 _LOG.debug("after format=%s", value) 

338 

339 if found and isinstance(value, Config): 

340 value = BpsConfig(value) 

341 

342 return found, value