Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of ctrl_bps. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22""" 

23Configuration class that adds order to searching sections for value, 

24expands environment variables and other config variables 

25""" 

26 

27from os.path import expandvars 

28import logging 

29import copy 

30import string 

31 

32from lsst.daf.butler.core.config import Config 

33 

34_LOG = logging.getLogger(__name__) 

35 

36 

37class BpsFormatter(string.Formatter): 

38 """String formatter class that allows BPS config 

39 search options 

40 """ 

41 def get_field(self, field_name, args, kwargs): 

42 _, val = args[0].search(field_name, opt=args[1]) 

43 return (val, field_name) 

44 

45 def get_value(self, key, args, kwargs): 

46 _, val = args[0].search(key, opt=args[1]) 

47 return val 

48 

49 

50class BpsConfig(Config): 

51 """Contains the configuration for a BPS submission. 

52 

53 Parameters 

54 ---------- 

55 other : `str`, `dict`, `Config`, `BpsConfig` 

56 Path to a yaml file or a dict/Config/BpsConfig containing configuration 

57 to copy. 

58 search_order : `list` of `str`, optional 

59 Root section names in the order in which they should be searched. 

60 """ 

61 def __init__(self, other, search_order=None): 

62 # In BPS config, the same setting can be defined multiple times in 

63 # different sections. The sections are search in a pre-defined 

64 # order. Hence, a value which is found first effectively overrides 

65 # values in later sections, if any. To achieve this goal, 

66 # the special methods __getitem__ and __contains__ were redefined to 

67 # use a custom search function internally. For this reason we can't 

68 # use super().__init__(other) as the super class defines its own 

69 # __getitem__ which is utilized during the initialization process ( 

70 # e.g. in expressions like self[<key>]). However, this function will 

71 # be overridden by the one defined here, in the subclass. Instead 

72 # we just initialize internal data structures and populate them 

73 # using the inherited update() method which does not rely on super 

74 # class __getitem__ method. 

75 super().__init__() 

76 try: 

77 config = Config(other) 

78 except RuntimeError: 

79 raise RuntimeError("A BpsConfig could not be loaded from other: %s" % other) 

80 self.update(config) 

81 

82 if isinstance(other, BpsConfig): 

83 self.search_order = copy.deepcopy(other.search_order) 

84 self.formatter = copy.deepcopy(other.formatter) 

85 else: 

86 if search_order is None: 

87 search_order = [] 

88 self.search_order = search_order 

89 self.formatter = BpsFormatter() 

90 

91 # Make sure search sections exist 

92 for key in self.search_order: 

93 if not Config.__contains__(self, key): 

94 self[key] = {} 

95 

96 def copy(self): 

97 """Makes a copy of config 

98 

99 Returns 

100 ------- 

101 copy : `~lsst.ctrl.bps.bps_config.BpsConfig` 

102 A duplicate of itself 

103 """ 

104 return BpsConfig(self) 

105 

106 def __getitem__(self, name): 

107 """Returns the value from the config for the given name 

108 

109 Parameters 

110 ---------- 

111 name : `str` 

112 Key to look for in config 

113 

114 Returns 

115 ------- 

116 val : `str`, `int`, `~lsst.ctrl.bps.bps_config.BPSConfig`, ... 

117 Value from config if found 

118 """ 

119 _, val = self.search(name, {}) 

120 

121 return val 

122 

123 def __contains__(self, name): 

124 """Checks whether name is in config. 

125 

126 Parameters 

127 ---------- 

128 name : `str` 

129 Key to look for in config. 

130 

131 Returns 

132 ------- 

133 found : `bool` 

134 Whether name was in config or not 

135 """ 

136 found, _ = self.search(name, {}) 

137 return found 

138 

139 def search(self, key, opt=None): 

140 """Searches for key using given opt following hierarchy rules. 

141 

142 Search hierarchy rules: current values, a given search object, and 

143 search order of config sections. 

144 

145 Parameters 

146 ---------- 

147 key : `str` 

148 Key to look for in config. 

149 opt : `dict`, optional 

150 Options dictionary to use while searching. All are optional. 

151 

152 ``"curvals"`` 

153 Means to pass in values for search order key 

154 (curr_<sectname>) or variable replacements. 

155 (`dict`, optional) 

156 ``"default"`` 

157 Value to return if not found. (`Any`, optional) 

158 ``"replaceVars"`` 

159 If search result is string, whether to replace variables 

160 inside it. By default set to True. (`bool`) 

161 ``"required"`` 

162 If replacing variables, whether to raise exception if 

163 variable is undefined. By default set to False. (`bool`) 

164 

165 Returns 

166 ------- 

167 found : `bool` 

168 Whether name was in config or not 

169 value : `str`, `int`, `BpsConfig`, ... 

170 Value from config if found 

171 """ 

172 _LOG.debug("search: initial key = '%s', opt = '%s'", key, opt) 

173 

174 if opt is None: 

175 opt = {} 

176 

177 found = False 

178 value = "" 

179 

180 # start with stored current values 

181 curvals = None 

182 if Config.__contains__(self, "current"): 

183 curvals = copy.deepcopy(Config.__getitem__(self, "current")) 

184 else: 

185 curvals = {} 

186 

187 # override with current values passed into function if given 

188 if "curvals" in opt: 

189 for ckey, cval in list(opt["curvals"].items()): 

190 _LOG.debug("using specified curval %s = %s", ckey, cval) 

191 curvals[ckey] = cval 

192 

193 _LOG.debug("curvals = %s", curvals) 

194 

195 if key in curvals: 

196 _LOG.debug("found %s in curvals", key) 

197 found = True 

198 value = curvals[key] 

199 elif "searchobj" in opt and key in opt["searchobj"]: 

200 found = True 

201 value = opt["searchobj"][key] 

202 else: 

203 for sect in self.search_order: 

204 if Config.__contains__(self, sect): 

205 _LOG.debug("Searching '%s' section for key '%s'", sect, key) 

206 search_sect = Config.__getitem__(self, sect) 

207 if "curr_" + sect in curvals: 

208 currkey = curvals["curr_" + sect] 

209 _LOG.debug("currkey for section %s = %s", sect, currkey) 

210 # search_sect = Config.__getitem__(search_sect, currkey) 

211 if Config.__contains__(search_sect, currkey): 

212 search_sect = Config.__getitem__(search_sect, currkey) 

213 

214 _LOG.debug("%s %s", key, search_sect) 

215 if Config.__contains__(search_sect, key): 

216 found = True 

217 value = Config.__getitem__(search_sect, key) 

218 break 

219 else: 

220 _LOG.debug("Missing search section '%s' while searching for '%s'", sect, key) 

221 

222 # lastly check root values 

223 if not found: 

224 _LOG.debug("Searching root section for key '%s'", key) 

225 if Config.__contains__(self, key): 

226 found = True 

227 value = Config.__getitem__(self, key) 

228 _LOG.debug("root value='%s'", value) 

229 

230 if not found and "default" in opt: 

231 value = opt["default"] 

232 found = True # ???? 

233 

234 if not found and opt.get("required", False): 

235 print("\n\nError: search for %s failed" % (key)) 

236 print("\tcurrent = ", self.get("current")) 

237 print("\topt = ", opt) 

238 print("\tcurvals = ", curvals) 

239 print("\n\n") 

240 raise KeyError("Error: Search failed (%s)" % key) 

241 

242 _LOG.debug("found=%s, value=%s", found, value) 

243 

244 _LOG.debug("opt=%s %s", opt, type(opt)) 

245 if found and isinstance(value, str) and opt.get("replaceVars", True): 

246 _LOG.debug("before format=%s", value) 

247 value = expandvars(value) # must replace env vars before calling format 

248 value = self.formatter.format(value, self, opt) 

249 _LOG.debug("after format=%s", value) 

250 

251 if found and isinstance(value, Config): 

252 value = BpsConfig(value) 

253 

254 return found, value