Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import abc 

23import click 

24from collections import defaultdict 

25import functools 

26import logging 

27import os 

28import traceback 

29import yaml 

30 

31from .cliLog import CliLog 

32from .opt import log_level_option, long_log_option, log_file_option, log_tty_option 

33from .progress import ClickProgressHandler 

34from lsst.utils import doImport 

35 

36 

37log = logging.getLogger(__name__) 

38 

39 

40@functools.lru_cache 

41def _importPlugin(pluginName): 

42 """Import a plugin that contains Click commands. 

43 

44 Parameters 

45 ---------- 

46 pluginName : `str` 

47 An importable module whose __all__ parameter contains the commands 

48 that can be called. 

49 

50 Returns 

51 ------- 

52 An imported module or None 

53 The imported module, or None if the module could not be imported. 

54 

55 Notes 

56 ----- 

57 A cache is used in order to prevent repeated reports of failure 

58 to import a module that can be triggered by ``butler --help``. 

59 """ 

60 try: 

61 return doImport(pluginName) 

62 except Exception as err: 

63 log.warning("Could not import plugin from %s, skipping.", pluginName) 

64 log.debug("Plugin import exception: %s\nTraceback:\n%s", err, 

65 "".join(traceback.format_tb(err.__traceback__))) 

66 return None 

67 

68 

69class LoaderCLI(click.MultiCommand, abc.ABC): 

70 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

71 subcommands at runtime.""" 

72 

73 def __init__(self, *args, **kwargs): 

74 super().__init__(*args, **kwargs) 

75 

76 @property 

77 @abc.abstractmethod 

78 def localCmdPkg(self): 

79 """localCmdPkg identifies the location of the commands that are in this 

80 package. `getLocalCommands` assumes that the commands can be found in 

81 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

82 should be overridden. 

83 

84 Returns 

85 ------- 

86 package : `str` 

87 The fully qualified location of this package. 

88 """ 

89 pass 

90 

91 def getLocalCommands(self): 

92 """Get the commands offered by the local package. This assumes that the 

93 commands can be found in `localCmdPkg.__all__`, if this is not the case 

94 then this function should be overridden. 

95 

96 Returns 

97 ------- 

98 commands : `defaultdict` [`str`, `list` [`str`]] 

99 The key is the command name. The value is a list of package(s) that 

100 contains the command. 

101 """ 

102 commandsLocation = _importPlugin(self.localCmdPkg) 

103 if commandsLocation is None: 

104 # _importPlugins logs an error, don't need to do it again here. 

105 return defaultdict(list) 

106 return defaultdict(list, {self._funcNameToCmdName(f): 

107 [self.localCmdPkg] for f in commandsLocation.__all__}) 

108 

109 def list_commands(self, ctx): 

110 """Used by Click to get all the commands that can be called by the 

111 butler command, it is used to generate the --help output. 

112 

113 Parameters 

114 ---------- 

115 ctx : `click.Context` 

116 The current Click context. 

117 

118 Returns 

119 ------- 

120 commands : `list` [`str`] 

121 The names of the commands that can be called by the butler command. 

122 """ 

123 self._setupLogging(ctx) 

124 commands = self._getCommands() 

125 self._raiseIfDuplicateCommands(commands) 

126 return sorted(commands) 

127 

128 def get_command(self, ctx, name): 

129 """Used by Click to get a single command for execution. 

130 

131 Parameters 

132 ---------- 

133 ctx : `click.Context` 

134 The current Click context. 

135 name : `str` 

136 The name of the command to return. 

137 

138 Returns 

139 ------- 

140 command : `click.Command` 

141 A Command that wraps a callable command function. 

142 """ 

143 self._setupLogging(ctx) 

144 commands = self._getCommands() 

145 if name not in commands: 

146 return None 

147 self._raiseIfDuplicateCommands(commands) 

148 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name)) 

149 

150 def _setupLogging(self, ctx): 

151 """Init the logging system and config it for the command. 

152 

153 Subcommands may further configure the log settings.""" 

154 if isinstance(ctx, click.Context): 

155 CliLog.initLog(longlog=ctx.params.get(long_log_option.name(), False), 

156 log_tty=ctx.params.get(log_tty_option.name(), True), 

157 log_file=ctx.params.get(log_file_option.name(), ())) 

158 if log_level_option.name() in ctx.params: 

159 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

160 else: 

161 # This works around a bug in sphinx-click, where it passes in the 

162 # click.MultiCommand instead of the context. 

163 # https://github.com/click-contrib/sphinx-click/issues/70 

164 CliLog.initLog(longlog=False) 

165 logging.debug("The passed-in context was not a click.Context, could not determine --long-log or " 

166 "--log-level values.") 

167 

168 @classmethod 

169 def getPluginList(cls): 

170 """Get the list of importable yaml files that contain cli data for this 

171 command. 

172 

173 Returns 

174 ------- 

175 `list` [`str`] 

176 The list of files that contain yaml data about a cli plugin. 

177 """ 

178 if not hasattr(cls, "pluginEnvVar"): 

179 return [] 

180 pluginModules = os.environ.get(cls.pluginEnvVar) 

181 if pluginModules: 

182 return [p for p in pluginModules.split(":") if p != ''] 

183 return [] 

184 

185 @classmethod 

186 def _funcNameToCmdName(cls, functionName): 

187 """Convert function name to the butler command name: change 

188 underscores, (used in functions) to dashes (used in commands), and 

189 change local-package command names that conflict with python keywords 

190 to a legal function name. 

191 """ 

192 return functionName.replace("_", "-") 

193 

194 @classmethod 

195 def _cmdNameToFuncName(cls, commandName): 

196 """Convert butler command name to function name: change dashes (used in 

197 commands) to underscores (used in functions), and for local-package 

198 commands names that conflict with python keywords, change the local, 

199 legal, function name to the command name.""" 

200 return commandName.replace("-", "_") 

201 

202 @staticmethod 

203 def _mergeCommandLists(a, b): 

204 """Combine two dicts whose keys are strings (command name) and values 

205 are list of string (the package(s) that provide the named command). 

206 

207 Parameters 

208 ---------- 

209 a : `defaultdict` [`str`, `list` [`str`]] 

210 The key is the command name. The value is a list of package(s) that 

211 contains the command. 

212 b : (same as a) 

213 

214 Returns 

215 ------- 

216 commands : `defaultdict` [`str`: [`str`]] 

217 For convenience, returns a extended with b. ('a' is modified in 

218 place.) 

219 """ 

220 for key, val in b.items(): 

221 a[key].extend(val) 

222 return a 

223 

224 @classmethod 

225 def _getPluginCommands(cls): 

226 """Get the commands offered by plugin packages. 

227 

228 Returns 

229 ------- 

230 commands : `defaultdict` [`str`, `list` [`str`]] 

231 The key is the command name. The value is a list of package(s) that 

232 contains the command. 

233 """ 

234 commands = defaultdict(list) 

235 for pluginName in cls.getPluginList(): 

236 try: 

237 with open(pluginName, "r") as resourceFile: 

238 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

239 except Exception as err: 

240 log.warning(f"Error loading commands from {pluginName}, skipping. {err}") 

241 continue 

242 if "cmd" not in resources: 

243 log.warning(f"No commands found in {pluginName}, skipping.") 

244 continue 

245 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

246 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

247 return commands 

248 

249 def _getCommands(self): 

250 """Get the commands offered by daf_butler and plugin packages. 

251 

252 Returns 

253 ------- 

254 commands : `defaultdict` [`str`, `list` [`str`]] 

255 The key is the command name. The value is a list of package(s) that 

256 contains the command. 

257 """ 

258 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

259 

260 @staticmethod 

261 def _raiseIfDuplicateCommands(commands): 

262 """If any provided command is offered by more than one package raise an 

263 exception. 

264 

265 Parameters 

266 ---------- 

267 commands : `defaultdict` [`str`, `list` [`str`]] 

268 The key is the command name. The value is a list of package(s) that 

269 contains the command. 

270 

271 Raises 

272 ------ 

273 click.ClickException 

274 Raised if a command is offered by more than one package, with an 

275 error message to be displayed to the user. 

276 """ 

277 

278 msg = "" 

279 for command, packages in commands.items(): 

280 if len(packages) > 1: 

281 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

282 if msg: 

283 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.") 

284 

285 

286class ButlerCLI(LoaderCLI): 

287 

288 localCmdPkg = "lsst.daf.butler.cli.cmd" 

289 

290 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

291 

292 @classmethod 

293 def _funcNameToCmdName(cls, functionName): 

294 # Docstring inherited from base class. 

295 

296 # The "import" command name and "butler_import" function name are 

297 # defined in cli/cmd/commands.py, and if those names are changed they 

298 # must be changed here as well. 

299 # It is expected that there will be very few butler command names that 

300 # need to be changed because of e.g. conflicts with python keywords (as 

301 # is done here and in _cmdNameToFuncName for the 'import' command). If 

302 # this becomes a common need then some way of doing this should be 

303 # invented that is better than hard coding the function names into 

304 # these conversion functions. An extension of the 'cli/resources.yaml' 

305 # file (as is currently used in obs_base) might be a good way to do it. 

306 if functionName == "butler_import": 

307 return "import" 

308 return super()._funcNameToCmdName(functionName) 

309 

310 @classmethod 

311 def _cmdNameToFuncName(cls, commandName): 

312 # Docstring inherited from base class. 

313 if commandName == "import": 

314 return "butler_import" 

315 return super()._cmdNameToFuncName(commandName) 

316 

317 

318@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

319@log_level_option() 

320@long_log_option() 

321@log_file_option() 

322@log_tty_option() 

323@ClickProgressHandler.option 

324def cli(log_level, long_log, log_file, log_tty, progress): 

325 # log_level is handled by get_command and list_commands, and is called in 

326 # one of those functions before this is called. long_log is handled by 

327 # setup_logging. 

328 pass 

329 

330 

331def main(): 

332 return cli()