Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Configuration class that adds order to searching sections for value, 

24expands environment variables and other config variables 

25""" 

26 

27from os.path import expandvars 

28import logging 

29import copy 

30import string 

31import re 

32 

33from lsst.daf.butler.core.config import Config 

34 

35_LOG = logging.getLogger(__name__) 

36 

37 

38class BpsFormatter(string.Formatter): 

39 """String formatter class that allows BPS config 

40 search options 

41 """ 

42 def get_field(self, field_name, args, kwargs): 

43 _, val = args[0].search(field_name, opt=args[1]) 

44 return (val, field_name) 

45 

46 def get_value(self, key, args, kwargs): 

47 _, val = args[0].search(key, opt=args[1]) 

48 return val 

49 

50 

51class BpsConfig(Config): 

52 """Contains the configuration for a BPS submission. 

53 

54 Parameters 

55 ---------- 

56 other : `str`, `dict`, `Config`, `BpsConfig` 

57 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

58 to copy. 

59 search_order : `list` of `str`, optional 

60 Root section names in the order in which they should be searched. 

61 """ 

62 def __init__(self, other, search_order=None): 

63 # In BPS config, the same setting can be defined multiple times in 

64 # different sections. The sections are search in a pre-defined 

65 # order. Hence, a value which is found first effectively overrides 

66 # values in later sections, if any. To achieve this goal, 

67 # the special methods __getitem__ and __contains__ were redefined to 

68 # use a custom search function internally. For this reason we can't 

69 # use super().__init__(other) as the super class defines its own 

70 # __getitem__ which is utilized during the initialization process ( 

71 # e.g. in expressions like self[<key>]). However, this function will 

72 # be overridden by the one defined here, in the subclass. Instead 

73 # we just initialize internal data structures and populate them 

74 # using the inherited update() method which does not rely on super 

75 # class __getitem__ method. 

76 super().__init__() 

77 try: 

78 config = Config(other) 

79 except RuntimeError: 

80 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other) 

81 self.update(config) 

82 

83 if isinstance(other, BpsConfig): 

84 self.search_order = copy.deepcopy(other.search_order) 

85 self.formatter = copy.deepcopy(other.formatter) 

86 else: 

87 if search_order is None: 

88 search_order = [] 

89 self.search_order = search_order 

90 self.formatter = BpsFormatter() 

91 

92 # Make sure search sections exist 

93 for key in self.search_order: 

94 if not Config.__contains__(self, key): 

95 self[key] = {} 

96 

97 def copy(self): 

98 """Makes a copy of config 

99 

100 Returns 

101 ------- 

102 copy : `~lsst.ctrl.bps.bps_config.BpsConfig` 

103 A duplicate of itself 

104 """ 

105 return BpsConfig(self) 

106 

107 def __getitem__(self, name): 

108 """Returns the value from the config for the given name 

109 

110 Parameters 

111 ---------- 

112 name : `str` 

113 Key to look for in config 

114 

115 Returns 

116 ------- 

117 val : `str`, `int`, `~lsst.ctrl.bps.bps_config.BPSConfig`, ... 

118 Value from config if found 

119 """ 

120 _, val = self.search(name, {}) 

121 

122 return val 

123 

124 def __contains__(self, name): 

125 """Checks whether name is in config. 

126 

127 Parameters 

128 ---------- 

129 name : `str` 

130 Key to look for in config. 

131 

132 Returns 

133 ------- 

134 found : `bool` 

135 Whether name was in config or not 

136 """ 

137 found, _ = self.search(name, {}) 

138 return found 

139 

140 def search(self, key, opt=None): 

141 """Searches for key using given opt following hierarchy rules. 

142 

143 Search hierarchy rules: current values, a given search object, and 

144 search order of config sections. 

145 

146 Parameters 

147 ---------- 

148 key : `str` 

149 Key to look for in config. 

150 opt : `dict`, optional 

151 Options dictionary to use while searching. All are optional. 

152 

153 ``"curvals"`` 

154 Means to pass in values for search order key 

155 (curr_<sectname>) or variable replacements. 

156 (`dict`, optional) 

157 ``"default"`` 

158 Value to return if not found. (`Any`, optional) 

159 ``"replaceEnvVars"`` 

160 If search result is string, whether to replace environment 

161 variables inside it with special placeholder (<ENV:name>). 

162 By default set to False. (`bool`) 

163 ``"expandEnvVars"`` 

164 If search result is string, whether to replace environment 

165 variables inside it with current environment value. 

166 By default set to False. (`bool`) 

167 ``"replaceVars"`` 

168 If search result is string, whether to replace variables 

169 inside it. By default set to True. (`bool`) 

170 ``"required"`` 

171 If replacing variables, whether to raise exception if 

172 variable is undefined. By default set to False. (`bool`) 

173 

174 Returns 

175 ------- 

176 found : `bool` 

177 Whether name was in config or not 

178 value : `str`, `int`, `BpsConfig`, ... 

179 Value from config if found 

180 """ 

181 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

182 

183 if opt is None: 

184 opt = {} 

185 

186 found = False 

187 value = "" 

188 

189 # start with stored current values 

190 curvals = None 

191 if Config.__contains__(self, "current"): 

192 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

193 else: 

194 curvals = {} 

195 

196 # override with current values passed into function if given 

197 if "curvals" in opt: 

198 for ckey, cval in list(opt["curvals"].items()): 

199 _LOG.debug("using specified curval %s = %s", ckey, cval) 

200 curvals[ckey] = cval 

201 

202 _LOG.debug("curvals = %s", curvals) 

203 

204 if key in curvals: 

205 _LOG.debug("found %s in curvals", key) 

206 found = True 

207 value = curvals[key] 

208 elif "searchobj" in opt and key in opt["searchobj"]: 

209 found = True 

210 value = opt["searchobj"][key] 

211 else: 

212 for sect in self.search_order: 

213 if Config.__contains__(self, sect): 

214 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

215 search_sect = Config.__getitem__(self, sect) 

216 if "curr_" + sect in curvals: 

217 currkey = curvals["curr_" + sect] 

218 _LOG.debug("currkey for section %s = %s", sect, currkey) 

219 # search_sect = Config.__getitem__(search_sect, currkey) 

220 if Config.__contains__(search_sect, currkey): 

221 search_sect = Config.__getitem__(search_sect, currkey) 

222 

223 _LOG.debug("%s %s", key, search_sect) 

224 if Config.__contains__(search_sect, key): 

225 found = True 

226 value = Config.__getitem__(search_sect, key) 

227 break 

228 else: 

229 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

230 

231 # lastly check root values 

232 if not found: 

233 _LOG.debug("Searching root section for key '%s'", key) 

234 if Config.__contains__(self, key): 

235 found = True 

236 value = Config.__getitem__(self, key) 

237 _LOG.debug("root value='%s'", value) 

238 

239 if not found and "default" in opt: 

240 value = opt["default"] 

241 found = True # ???? 

242 

243 if not found and opt.get("required", False): 

244 print("\n\nError: search for %s failed" % (key)) 

245 print("\tcurrent = ", self.get("current")) 

246 print("\topt = ", opt) 

247 print("\tcurvals = ", curvals) 

248 print("\n\n") 

249 raise KeyError("Error: Search failed (%s)" % key) 

250 

251 _LOG.debug("found=%s, value=%s", found, value) 

252 

253 _LOG.debug("opt=%s %s", opt, type(opt)) 

254 if found and isinstance(value, str): 

255 if opt.get("expandEnvVars", True): 

256 _LOG.debug("before format=%s", value) 

257 value = re.sub(r"<ENV:([^>]+)>", r"$\1", value) 

258 value = expandvars(value) 

259 elif opt.get("replaceEnvVars", False): 

260 value = re.sub(r"\${([^}]+)}", r"<ENV:\1>", value) 

261 value = re.sub(r"\$(\S+)", r"<ENV:\1>", value) 

262 

263 if opt.get("replaceVars", True): 

264 # default only applies to original search key 

265 default = opt.pop("default", None) 

266 

267 # Temporarily replace any env vars so formatter doesn't try to replace them. 

268 value = re.sub(r"\${([^}]+)}", r"<BPSTMP:\1>", value) 

269 

270 value = self.formatter.format(value, self, opt) 

271 

272 # Replace any temporary env place holders. 

273 value = re.sub(r"<BPSTMP:([^>]+)>", r"${\1}", value) 

274 

275 if default: # reset to avoid modifying dict parameter. 

276 opt["default"] = default 

277 

278 _LOG.debug("after format=%s", value) 

279 

280 if found and isinstance(value, Config): 

281 value = BpsConfig(value) 

282 

283 return found, value