Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Configuration class that adds order to searching sections for value, 

24expands environment variables and other config variables 

25""" 

26 

27from os.path import expandvars 

28import logging 

29import copy 

30import string 

31 

32from lsst.daf.butler.core.config import Config 

33 

34_LOG = logging.getLogger() 

35 

36 

37class BpsFormatter(string.Formatter): 

38 """String formatter class that allows BPS config 

39 search options 

40 """ 

41 def get_field(self, field_name, args, kwargs): 

42 _, val = args[0].search(field_name, opt=args[1]) 

43 return (val, field_name) 

44 

45 def get_value(self, key, args, kwargs): 

46 _, val = args[0].search(key, opt=args[1]) 

47 return val 

48 

49 

50class BpsConfig(Config): 

51 """Contains the configuration for a BPS submission. 

52 

53 Parameters 

54 ---------- 

55 other: `str`, `dict`, `Config`, `BpsConfig` 

56 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

57 to copy. 

58 """ 

59 def __init__(self, other, search_order=None): 

60 # In BPS config, the same setting can be defined multiple times in 

61 # different sections. The sections are search in a pre-defined 

62 # order. Hence, a value which is found first effectively overrides 

63 # values in later sections, if any. To achieve this goal, 

64 # the special methods __getitem__ and __contains__ were redefined to 

65 # use a custom search function internally. For this reason we can't 

66 # use super().__init__(other) as the super class defines its own 

67 # __getitem__ which is utilized during the initialization process ( 

68 # e.g. in expressions like self[<key>]). However, this function will 

69 # be overridden by the one defined here, in the subclass. Instead 

70 # we just initialize internal data structures and populate them 

71 # using the inherited update() method which does not rely on super 

72 # class __getitem__ method. 

73 super().__init__() 

74 try: 

75 config = Config(other) 

76 except RuntimeError: 

77 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other) 

78 self.update(config) 

79 

80 if isinstance(other, BpsConfig): 

81 self.search_order = copy.deepcopy(other.search_order) 

82 self.formatter = copy.deepcopy(other.formatter) 

83 else: 

84 if search_order is None: 

85 search_order = [] 

86 self.search_order = search_order 

87 self.formatter = BpsFormatter() 

88 

89 def copy(self): 

90 """Makes a copy of config 

91 

92 Returns 

93 ------- 

94 copy: `BpsConfig` 

95 A duplicate of itself 

96 """ 

97 return BpsConfig(self) 

98 

99 def __getitem__(self, name): 

100 """Returns the value from the config for the given name 

101 

102 Parameters 

103 ---------- 

104 name: `str` 

105 Key to look for in config 

106 

107 Returns 

108 ------- 

109 val: `str`, `int`, `BPSConfig`, ... 

110 Value from config if found 

111 """ 

112 _, val = self.search(name, {}) 

113 

114 return val 

115 

116 def __contains__(self, name): 

117 """Checks whether name is in config 

118 

119 Parameters 

120 ---------- 

121 name: `str` 

122 Key to look for in config 

123 

124 Returns 

125 ------- 

126 found: `bool` 

127 Whether name was in config or not 

128 """ 

129 found, _ = self.search(name, {}) 

130 return found 

131 

132 def search(self, key, opt=None): 

133 """Searches for key using given opt following hierarchy rules. 

134 

135 Search hierarchy rules: current values, a given search object, and 

136 search order of config sections. 

137 

138 Parameters 

139 ---------- 

140 key: `str` 

141 Key to look for in config. 

142 opt: `dict`, optional 

143 Options dictionary to use while searching. All are optional. 

144 

145 ``"curvals"`` 

146 Means to pass in values for search order key 

147 (curr_<sectname>) or variable replacements. 

148 (`dict`, optional) 

149 ``"default"`` 

150 Value to return if not found. (`Any`, optional) 

151 ``"replaceVars"`` 

152 If search result is string, whether to replace variables 

153 inside it. By default set to True. (`bool`) 

154 ``"required"`` 

155 If replacing variables, whether to raise exception if 

156 variable is undefined. By default set to False. (`bool`) 

157 

158 Returns 

159 ------- 

160 found: `bool` 

161 Whether name was in config or not 

162 value: `str`, `int`, `BpsConfig`, ... 

163 Value from config if found 

164 """ 

165 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

166 

167 if opt is None: 

168 opt = {} 

169 

170 found = False 

171 value = "" 

172 

173 # start with stored current values 

174 curvals = None 

175 if Config.__contains__(self, "current"): 

176 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

177 else: 

178 curvals = {} 

179 

180 # override with current values passed into function if given 

181 if "curvals" in opt: 

182 for ckey, cval in list(opt["curvals"].items()): 

183 _LOG.debug("using specified curval %s = %s", ckey, cval) 

184 curvals[ckey] = cval 

185 

186 _LOG.debug("curvals = %s", curvals) 

187 

188 if key in curvals: 

189 _LOG.debug("found %s in curvals", key) 

190 found = True 

191 value = curvals[key] 

192 elif "searchobj" in opt and key in opt["searchobj"]: 

193 found = True 

194 value = opt["searchobj"][key] 

195 else: 

196 for sect in self.search_order: 

197 if Config.__contains__(self, sect): 

198 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

199 search_sect = Config.__getitem__(self, sect) 

200 if "curr_" + sect in curvals: 

201 currkey = curvals["curr_" + sect] 

202 _LOG.debug("currkey for section %s = %s", sect, currkey) 

203 # search_sect = Config.__getitem__(search_sect, currkey) 

204 if Config.__contains__(search_sect, currkey): 

205 search_sect = Config.__getitem__(search_sect, currkey) 

206 

207 _LOG.debug("%s %s", key, search_sect) 

208 if Config.__contains__(search_sect, key): 

209 found = True 

210 value = Config.__getitem__(search_sect, key) 

211 break 

212 else: 

213 _LOG.warning("Missing search section '%s' while searching for '%s'", sect, key) 

214 

215 # lastly check root values 

216 if not found: 

217 _LOG.debug("Searching root section for key '%s'", key) 

218 if Config.__contains__(self, key): 

219 found = True 

220 value = Config.__getitem__(self, key) 

221 _LOG.debug("root value='%s'", value) 

222 

223 if not found and "default" in opt: 

224 value = opt["default"] 

225 found = True # ???? 

226 

227 if not found and opt.get("required", False): 

228 print("\n\nError: search for %s failed" % (key)) 

229 print("\tcurrent = ", self.get("current")) 

230 print("\topt = ", opt) 

231 print("\tcurvals = ", curvals) 

232 print("\n\n") 

233 raise KeyError("Error: Search failed (%s)" % key) 

234 

235 _LOG.debug("found=%s, value=%s", found, value) 

236 

237 _LOG.debug("opt=%s %s", opt, type(opt)) 

238 if found and isinstance(value, str) and opt.get("replaceVars", True): 

239 _LOG.debug("before format=%s", value) 

240 value = expandvars(value) # must replace env vars before calling format 

241 value = self.formatter.format(value, self, opt) 

242 _LOG.debug("after format=%s", value) 

243 

244 if found and isinstance(value, Config): 

245 value = BpsConfig(value) 

246 

247 return found, value