Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import abc 

23import click 

24from collections import defaultdict 

25import logging 

26import os 

27import yaml 

28 

29from .cliLog import CliLog 

30from .opt import log_level_option 

31from lsst.utils import doImport 

32 

33 

34log = logging.getLogger(__name__) 

35 

36LONG_LOG_FLAG = "--long-log" 

37 

38 

39class LoaderCLI(click.MultiCommand, abc.ABC): 

40 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

41 subcommands at runtime.""" 

42 

43 def __init__(self, *args, **kwargs): 

44 super().__init__(*args, **kwargs) 

45 

46 @property 

47 @abc.abstractmethod 

48 def localCmdPkg(self): 

49 """localCmdPkg identifies the location of the commands that are in this 

50 package. `getLocalCommands` assumes that the commands can be found in 

51 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

52 should be overrideen. 

53 

54 Returns 

55 ------- 

56 package : `str` 

57 The fully qualified location of this package. 

58 """ 

59 pass 

60 

61 def getLocalCommands(self): 

62 """Get the commands offered by the local package. This assumes that the 

63 commands can be found in `localCmdPkg.__all__`, if this is not the case 

64 then this function should be overrideen. 

65 

66 Returns 

67 ------- 

68 commands : `defaultdict` [`str`, `list` [`str`]] 

69 The key is the command name. The value is a list of package(s) that 

70 contains the command. 

71 """ 

72 commandsLocation = self._importPlugin(self.localCmdPkg) 

73 if commandsLocation is None: 

74 # _importPlugins logs an error, don't need to do it again here. 

75 return {} 

76 return defaultdict(list, {self._funcNameToCmdName(f): 

77 [self.localCmdPkg] for f in commandsLocation.__all__}) 

78 

79 def list_commands(self, ctx): 

80 """Used by Click to get all the commands that can be called by the 

81 butler command, it is used to generate the --help output. 

82 

83 Parameters 

84 ---------- 

85 ctx : `click.Context` 

86 The current Click context. 

87 

88 Returns 

89 ------- 

90 commands : `list` [`str`] 

91 The names of the commands that can be called by the butler command. 

92 """ 

93 self._setupLogging(ctx) 

94 commands = self._getCommands() 

95 self._raiseIfDuplicateCommands(commands) 

96 return sorted(commands) 

97 

98 def get_command(self, ctx, name): 

99 """Used by Click to get a single command for execution. 

100 

101 Parameters 

102 ---------- 

103 ctx : `click.Context` 

104 The current Click context. 

105 name : `str` 

106 The name of the command to return. 

107 

108 Returns 

109 ------- 

110 command : `click.Command` 

111 A Command that wraps a callable command function. 

112 """ 

113 self._setupLogging(ctx) 

114 commands = self._getCommands() 

115 if name not in commands: 

116 return None 

117 self._raiseIfDuplicateCommands(commands) 

118 return self._importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name)) 

119 

120 def _setupLogging(self, ctx): 

121 """Init the logging system and config it for the command. 

122 

123 Subcommands may further configure the log settings.""" 

124 CliLog.initLog(longlog=LONG_LOG_FLAG in ctx.params) 

125 if log_level_option.optionKey in ctx.params: 

126 CliLog.setLogLevels(ctx.params[log_level_option.optionKey]) 

127 

128 @staticmethod 

129 def getPluginList(): 

130 """Get the list of importable yaml files that contain cli data for this 

131 command. 

132 

133 Returns 

134 ------- 

135 `list` [`str`] 

136 The list of files that contain yaml data about a cli plugin. 

137 """ 

138 return [] 

139 

140 @classmethod 

141 def _funcNameToCmdName(cls, functionName): 

142 """Convert function name to the butler command name: change 

143 underscores, (used in functions) to dashes (used in commands), and 

144 change local-package command names that conflict with python keywords 

145 to a legal function name. 

146 """ 

147 return functionName.replace("_", "-") 

148 

149 @classmethod 

150 def _cmdNameToFuncName(cls, commandName): 

151 """Convert butler command name to function name: change dashes (used in 

152 commands) to underscores (used in functions), and for local-package 

153 commands names that conflict with python keywords, change the local, 

154 legal, function name to the command name.""" 

155 return commandName.replace("-", "_") 

156 

157 @staticmethod 

158 def _importPlugin(pluginName): 

159 """Import a plugin that contains Click commands. 

160 

161 Parameters 

162 ---------- 

163 pluginName : `str` 

164 An importable module whose __all__ parameter contains the commands 

165 that can be called. 

166 

167 Returns 

168 ------- 

169 An imported module or None 

170 The imported module, or None if the module could not be imported. 

171 """ 

172 try: 

173 return doImport(pluginName) 

174 except Exception as err: 

175 log.warning("Could not import plugin from %s, skipping.", pluginName) 

176 log.debug("Plugin import exception: %s", err) 

177 return None 

178 

179 @staticmethod 

180 def _mergeCommandLists(a, b): 

181 """Combine two dicts whose keys are strings (command name) and values 

182 are list of string (the package(s) that provide the named command). 

183 

184 Parameters 

185 ---------- 

186 a : `defaultdict` [`str`, `list` [`str`]] 

187 The key is the command name. The value is a list of package(s) that 

188 contains the command. 

189 b : (same as a) 

190 

191 Returns 

192 ------- 

193 commands : `defaultdict` [`str`: [`str`]] 

194 For convenience, returns a extended with b. ('a' is modified in 

195 place.) 

196 """ 

197 for key, val in b.items(): 

198 a[key].extend(val) 

199 return a 

200 

201 @classmethod 

202 def _getPluginCommands(cls): 

203 """Get the commands offered by plugin packages. 

204 

205 Returns 

206 ------- 

207 commands : `defaultdict` [`str`, `list` [`str`]] 

208 The key is the command name. The value is a list of package(s) that 

209 contains the command. 

210 """ 

211 commands = defaultdict(list) 

212 for pluginName in cls.getPluginList(): 

213 try: 

214 with open(pluginName, "r") as resourceFile: 

215 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

216 except Exception as err: 

217 log.warning(f"Error loading commands from {pluginName}, skipping. {err}") 

218 continue 

219 if "cmd" not in resources: 

220 log.warning(f"No commands found in {pluginName}, skipping.") 

221 continue 

222 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

223 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

224 return commands 

225 

226 def _getCommands(self): 

227 """Get the commands offered by daf_butler and plugin packages. 

228 

229 Returns 

230 ------- 

231 commands : `defaultdict` [`str`, `list` [`str`]] 

232 The key is the command name. The value is a list of package(s) that 

233 contains the command. 

234 """ 

235 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

236 

237 @staticmethod 

238 def _raiseIfDuplicateCommands(commands): 

239 """If any provided command is offered by more than one package raise an 

240 exception. 

241 

242 Parameters 

243 ---------- 

244 commands : `defaultdict` [`str`, `list` [`str`]] 

245 The key is the command name. The value is a list of package(s) that 

246 contains the command. 

247 

248 Raises 

249 ------ 

250 click.ClickException 

251 Raised if a command is offered by more than one package, with an 

252 error message to be displayed to the user. 

253 """ 

254 

255 msg = "" 

256 for command, packages in commands.items(): 

257 if len(packages) > 1: 

258 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

259 if msg: 

260 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.") 

261 

262 

263class ButlerCLI(LoaderCLI): 

264 

265 localCmdPkg = "lsst.daf.butler.cli.cmd" 

266 

267 @classmethod 

268 def _funcNameToCmdName(cls, functionName): 

269 # Docstring inherited from base class. 

270 

271 # The "import" command name and "butler_import" function name are 

272 # defined in cli/cmd/commands.py, and if those names are changed they 

273 # must be changed here as well. 

274 # It is expected that there will be very few butler command names that 

275 # need to be changed because of e.g. conflicts with python keywords (as 

276 # is done here and in _cmdNameToFuncName for the 'import' command). If 

277 # this becomes a common need then some way of doing this should be 

278 # invented that is better than hard coding the function names into 

279 # these conversion functions. An extension of the 'cli/resources.yaml' 

280 # file (as is currently used in obs_base) might be a good way to do it. 

281 if functionName == "butler_import": 

282 return "import" 

283 return super()._funcNameToCmdName(functionName) 

284 

285 @classmethod 

286 def _cmdNameToFuncName(cls, commandName): 

287 # Docstring inherited from base class. 

288 if commandName == "import": 

289 return "butler_import" 

290 return super()._cmdNameToFuncName(commandName) 

291 

292 @staticmethod 

293 def getPluginList(): 

294 # Docstring inherited from base class. 

295 pluginModules = os.environ.get("DAF_BUTLER_PLUGINS") 

296 if pluginModules: 

297 return [p for p in pluginModules.split(":") if p != ''] 

298 return [] 

299 

300 

301@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

302@log_level_option() 

303def cli(log_level): 

304 # log_level is handled by get_command and list_commands, and is called in 

305 # one of those functions before this is called. 

306 pass 

307 

308 

309def main(): 

310 return cli()