Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Configuration class that adds order to searching sections for value, 

23expands environment variables and other config variables. 

24""" 

25 

26__all__ = ["BPS_SEARCH_ORDER", "BpsConfig", "BpsFormatter"] 

27 

28 

29from os.path import expandvars 

30import logging 

31import copy 

32import string 

33import re 

34 

35from lsst.daf.butler.core.config import Config 

36 

37 

38_LOG = logging.getLogger(__name__) 

39 

40BPS_SEARCH_ORDER = ["payload", "pipetask", "site", "bps_defined"] 

41 

42 

43class BpsFormatter(string.Formatter): 

44 """String formatter class that allows BPS config search options. 

45 """ 

46 def get_field(self, field_name, args, kwargs): 

47 _, val = args[0].search(field_name, opt=args[1]) 

48 return val, field_name 

49 

50 def get_value(self, key, args, kwargs): 

51 _, val = args[0].search(key, opt=args[1]) 

52 return val 

53 

54 

55class BpsConfig(Config): 

56 """Contains the configuration for a BPS submission. 

57 

58 Parameters 

59 ---------- 

60 other : `str`, `dict`, `Config`, `BpsConfig` 

61 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

62 to copy. 

63 search_order : `list` [`str`], optional 

64 Root section names in the order in which they should be searched. 

65 """ 

66 def __init__(self, other, search_order=None): 

67 # In BPS config, the same setting can be defined multiple times in 

68 # different sections. The sections are search in a pre-defined 

69 # order. Hence, a value which is found first effectively overrides 

70 # values in later sections, if any. To achieve this goal, 

71 # the special methods __getitem__ and __contains__ were redefined to 

72 # use a custom search function internally. For this reason we can't 

73 # use super().__init__(other) as the super class defines its own 

74 # __getitem__ which is utilized during the initialization process ( 

75 # e.g. in expressions like self[<key>]). However, this function will 

76 # be overridden by the one defined here, in the subclass. Instead 

77 # we just initialize internal data structures and populate them 

78 # using the inherited update() method which does not rely on super 

79 # class __getitem__ method. 

80 super().__init__() 

81 try: 

82 config = Config(other) 

83 except RuntimeError: 

84 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other) 

85 self.update(config) 

86 

87 if isinstance(other, BpsConfig): 

88 self.search_order = copy.deepcopy(other.search_order) 

89 self.formatter = copy.deepcopy(other.formatter) 

90 else: 

91 if search_order is None: 

92 search_order = [] 

93 self.search_order = search_order 

94 self.formatter = BpsFormatter() 

95 

96 # Make sure search sections exist 

97 for key in self.search_order: 

98 if not Config.__contains__(self, key): 

99 self[key] = {} 

100 

101 def copy(self): 

102 """Make a copy of config. 

103 

104 Returns 

105 ------- 

106 copy : `lsst.ctrl.bps.BpsConfig` 

107 A duplicate of itself. 

108 """ 

109 return BpsConfig(self) 

110 

111 def __getitem__(self, name): 

112 """Return the value from the config for the given name. 

113 

114 Parameters 

115 ---------- 

116 name : `str` 

117 Key to look for in config 

118 

119 Returns 

120 ------- 

121 val : `str`, `int`, `lsst.ctrl.bps.BPSConfig`, ... 

122 Value from config if found. 

123 """ 

124 _, val = self.search(name, {}) 

125 

126 return val 

127 

128 def __contains__(self, name): 

129 """Check whether name is in config. 

130 

131 Parameters 

132 ---------- 

133 name : `str` 

134 Key to look for in config. 

135 

136 Returns 

137 ------- 

138 found : `bool` 

139 Whether name was in config or not. 

140 """ 

141 found, _ = self.search(name, {}) 

142 return found 

143 

144 def search(self, key, opt=None): 

145 """Search for key using given opt following hierarchy rules. 

146 

147 Search hierarchy rules: current values, a given search object, and 

148 search order of config sections. 

149 

150 Parameters 

151 ---------- 

152 key : `str` 

153 Key to look for in config. 

154 opt : `dict` [`str`, `Any`], optional 

155 Options dictionary to use while searching. All are optional. 

156 

157 ``"curvals"`` 

158 Means to pass in values for search order key 

159 (curr_<sectname>) or variable replacements. 

160 (`dict`, optional) 

161 ``"default"`` 

162 Value to return if not found. (`Any`, optional) 

163 ``"replaceEnvVars"`` 

164 If search result is string, whether to replace environment 

165 variables inside it with special placeholder (<ENV:name>). 

166 By default set to False. (`bool`) 

167 ``"expandEnvVars"`` 

168 If search result is string, whether to replace environment 

169 variables inside it with current environment value. 

170 By default set to False. (`bool`) 

171 ``"replaceVars"`` 

172 If search result is string, whether to replace variables 

173 inside it. By default set to True. (`bool`) 

174 ``"required"`` 

175 If replacing variables, whether to raise exception if 

176 variable is undefined. By default set to False. (`bool`) 

177 

178 Returns 

179 ------- 

180 found : `bool` 

181 Whether name was in config or not. 

182 value : `str`, `int`, `lsst.ctrl.bps.BpsConfig`, ... 

183 Value from config if found. 

184 """ 

185 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

186 

187 if opt is None: 

188 opt = {} 

189 

190 found = False 

191 value = "" 

192 

193 # start with stored current values 

194 curvals = None 

195 if Config.__contains__(self, "current"): 

196 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

197 else: 

198 curvals = {} 

199 

200 # override with current values passed into function if given 

201 if "curvals" in opt: 

202 for ckey, cval in list(opt["curvals"].items()): 

203 _LOG.debug("using specified curval %s = %s", ckey, cval) 

204 curvals[ckey] = cval 

205 

206 _LOG.debug("curvals = %s", curvals) 

207 

208 if key in curvals: 

209 _LOG.debug("found %s in curvals", key) 

210 found = True 

211 value = curvals[key] 

212 elif "searchobj" in opt and key in opt["searchobj"]: 

213 found = True 

214 value = opt["searchobj"][key] 

215 else: 

216 for sect in self.search_order: 

217 if Config.__contains__(self, sect): 

218 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

219 search_sect = Config.__getitem__(self, sect) 

220 if "curr_" + sect in curvals: 

221 currkey = curvals["curr_" + sect] 

222 _LOG.debug("currkey for section %s = %s", sect, currkey) 

223 if Config.__contains__(search_sect, currkey): 

224 search_sect = Config.__getitem__(search_sect, currkey) 

225 

226 _LOG.debug("%s %s", key, search_sect) 

227 if Config.__contains__(search_sect, key): 

228 found = True 

229 value = Config.__getitem__(search_sect, key) 

230 break 

231 else: 

232 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

233 

234 # lastly check root values 

235 if not found: 

236 _LOG.debug("Searching root section for key '%s'", key) 

237 if Config.__contains__(self, key): 

238 found = True 

239 value = Config.__getitem__(self, key) 

240 _LOG.debug("root value='%s'", value) 

241 

242 if not found and "default" in opt: 

243 value = opt["default"] 

244 found = True # ???? 

245 

246 if not found and opt.get("required", False): 

247 print("\n\nError: search for %s failed" % (key)) 

248 print("\tcurrent = ", self.get("current")) 

249 print("\topt = ", opt) 

250 print("\tcurvals = ", curvals) 

251 print("\n\n") 

252 raise KeyError("Error: Search failed (%s)" % key) 

253 

254 _LOG.debug("found=%s, value=%s", found, value) 

255 

256 _LOG.debug("opt=%s %s", opt, type(opt)) 

257 if found and isinstance(value, str): 

258 if opt.get("expandEnvVars", True): 

259 _LOG.debug("before format=%s", value) 

260 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

261 value = expandvars(value) 

262 elif opt.get("replaceEnvVars", False): 

263 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

264 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

265 

266 if opt.get("replaceVars", True): 

267 # default only applies to original search key 

268 default = opt.pop("default", None) 

269 

270 # Temporarily replace any env vars so formatter doesn't try to 

271 # replace them. 

272 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

273 

274 value = self.formatter.format(value, self, opt) 

275 

276 # Replace any temporary env place holders. 

277 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

278 

279 if default: # reset to avoid modifying dict parameter. 

280 opt["default"] = default 

281 

282 _LOG.debug("after format=%s", value) 

283 

284 if found and isinstance(value, Config): 

285 value = BpsConfig(value) 

286 

287 return found, value