Coverage for python/lsst/daf/butler/cli/butler.py: 34%

132 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-02-01 02:05 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "LoaderCLI", 

25 "ButlerCLI", 

26 "cli", 

27 "main", 

28) 

29 

30 

31import abc 

32import functools 

33import logging 

34import os 

35import traceback 

36import types 

37from collections import defaultdict 

38from typing import Any 

39 

40import click 

41import yaml 

42from lsst.utils import doImport 

43 

44from .cliLog import CliLog 

45from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

46from .progress import ClickProgressHandler 

47 

48log = logging.getLogger(__name__) 

49 

50 

51@functools.lru_cache 

52def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command: 

53 """Import a plugin that contains Click commands. 

54 

55 Parameters 

56 ---------- 

57 pluginName : `str` 

58 An importable module whose __all__ parameter contains the commands 

59 that can be called. 

60 

61 Returns 

62 ------- 

63 An imported module or None 

64 The imported module, or None if the module could not be imported. 

65 

66 Notes 

67 ----- 

68 A cache is used in order to prevent repeated reports of failure 

69 to import a module that can be triggered by ``butler --help``. 

70 """ 

71 try: 

72 return doImport(pluginName) 

73 except Exception as err: 

74 log.warning("Could not import plugin from %s, skipping.", pluginName) 

75 log.debug( 

76 "Plugin import exception: %s\nTraceback:\n%s", 

77 err, 

78 "".join(traceback.format_tb(err.__traceback__)), 

79 ) 

80 return None 

81 

82 

83class LoaderCLI(click.MultiCommand, abc.ABC): 

84 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

85 subcommands at runtime.""" 

86 

87 def __init__(self, *args: Any, **kwargs: Any) -> None: 

88 super().__init__(*args, **kwargs) 

89 

90 @property 

91 @abc.abstractmethod 

92 def localCmdPkg(self) -> str: 

93 """localCmdPkg identifies the location of the commands that are in this 

94 package. `getLocalCommands` assumes that the commands can be found in 

95 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

96 should be overridden. 

97 

98 Returns 

99 ------- 

100 package : `str` 

101 The fully qualified location of this package. 

102 """ 

103 raise NotImplementedError() 

104 

105 def getLocalCommands(self) -> defaultdict[str, list[str]]: 

106 """Get the commands offered by the local package. This assumes that the 

107 commands can be found in `localCmdPkg.__all__`, if this is not the case 

108 then this function should be overridden. 

109 

110 Returns 

111 ------- 

112 commands : `defaultdict` [`str`, `list` [`str`]] 

113 The key is the command name. The value is a list of package(s) that 

114 contains the command. 

115 """ 

116 commandsLocation = _importPlugin(self.localCmdPkg) 

117 if commandsLocation is None: 

118 # _importPlugins logs an error, don't need to do it again here. 

119 return defaultdict(list) 

120 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}" 

121 return defaultdict( 

122 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

123 ) 

124 

125 def list_commands(self, ctx: click.Context) -> list[str]: 

126 """Used by Click to get all the commands that can be called by the 

127 butler command, it is used to generate the --help output. 

128 

129 Parameters 

130 ---------- 

131 ctx : `click.Context` 

132 The current Click context. 

133 

134 Returns 

135 ------- 

136 commands : `list` [`str`] 

137 The names of the commands that can be called by the butler command. 

138 """ 

139 self._setupLogging(ctx) 

140 commands = self._getCommands() 

141 self._raiseIfDuplicateCommands(commands) 

142 return sorted(commands) 

143 

144 def get_command(self, ctx: click.Context, name: str) -> click.Command | None: 

145 """Used by Click to get a single command for execution. 

146 

147 Parameters 

148 ---------- 

149 ctx : `click.Context` 

150 The current Click context. 

151 name : `str` 

152 The name of the command to return. 

153 

154 Returns 

155 ------- 

156 command : `click.Command` 

157 A Command that wraps a callable command function. 

158 """ 

159 self._setupLogging(ctx) 

160 commands = self._getCommands() 

161 if name not in commands: 

162 return None 

163 self._raiseIfDuplicateCommands(commands) 

164 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name) 

165 # The click.command decorator returns an instance of a class, which 

166 # is something that doImport is not expecting. We add it in as an 

167 # option here to appease mypy. 

168 plugin = _importPlugin(module_str) 

169 if not plugin: 

170 return None 

171 if not isinstance(plugin, click.Command): 

172 raise RuntimeError( 

173 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}" 

174 ) 

175 return plugin 

176 

177 def _setupLogging(self, ctx: click.Context | None) -> None: 

178 """Init the logging system and config it for the command. 

179 

180 Subcommands may further configure the log settings.""" 

181 if isinstance(ctx, click.Context): 

182 CliLog.initLog( 

183 longlog=ctx.params.get(long_log_option.name(), False), 

184 log_tty=ctx.params.get(log_tty_option.name(), True), 

185 log_file=ctx.params.get(log_file_option.name(), ()), 

186 log_label=ctx.params.get(log_label_option.name(), ()), 

187 ) 

188 if log_level_option.name() in ctx.params: 

189 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

190 else: 

191 # This works around a bug in sphinx-click, where it passes in the 

192 # click.MultiCommand instead of the context. 

193 # https://github.com/click-contrib/sphinx-click/issues/70 

194 CliLog.initLog(longlog=False) 

195 logging.debug( 

196 "The passed-in context was not a click.Context, could not determine --long-log or " 

197 "--log-level values." 

198 ) 

199 

200 @classmethod 

201 def getPluginList(cls) -> list[str]: 

202 """Get the list of importable yaml files that contain cli data for this 

203 command. 

204 

205 Returns 

206 ------- 

207 `list` [`str`] 

208 The list of files that contain yaml data about a cli plugin. 

209 """ 

210 if not hasattr(cls, "pluginEnvVar"): 

211 return [] 

212 pluginModules = os.environ.get(cls.pluginEnvVar) 

213 if pluginModules: 

214 return [p for p in pluginModules.split(":") if p != ""] 

215 return [] 

216 

217 @classmethod 

218 def _funcNameToCmdName(cls, functionName: str) -> str: 

219 """Convert function name to the butler command name: change 

220 underscores, (used in functions) to dashes (used in commands), and 

221 change local-package command names that conflict with python keywords 

222 to a legal function name. 

223 """ 

224 return functionName.replace("_", "-") 

225 

226 @classmethod 

227 def _cmdNameToFuncName(cls, commandName: str) -> str: 

228 """Convert butler command name to function name: change dashes (used in 

229 commands) to underscores (used in functions), and for local-package 

230 commands names that conflict with python keywords, change the local, 

231 legal, function name to the command name.""" 

232 return commandName.replace("-", "_") 

233 

234 @staticmethod 

235 def _mergeCommandLists( 

236 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]] 

237 ) -> defaultdict[str, list[str]]: 

238 """Combine two dicts whose keys are strings (command name) and values 

239 are list of string (the package(s) that provide the named command). 

240 

241 Parameters 

242 ---------- 

243 a : `defaultdict` [`str`, `list` [`str`]] 

244 The key is the command name. The value is a list of package(s) that 

245 contains the command. 

246 b : (same as a) 

247 

248 Returns 

249 ------- 

250 commands : `defaultdict` [`str`: [`str`]] 

251 For convenience, returns a extended with b. ('a' is modified in 

252 place.) 

253 """ 

254 for key, val in b.items(): 

255 a[key].extend(val) 

256 return a 

257 

258 @classmethod 

259 def _getPluginCommands(cls) -> defaultdict[str, list[str]]: 

260 """Get the commands offered by plugin packages. 

261 

262 Returns 

263 ------- 

264 commands : `defaultdict` [`str`, `list` [`str`]] 

265 The key is the command name. The value is a list of package(s) that 

266 contains the command. 

267 """ 

268 commands: defaultdict[str, list[str]] = defaultdict(list) 

269 for pluginName in cls.getPluginList(): 

270 try: 

271 with open(pluginName, "r") as resourceFile: 

272 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

273 except Exception as err: 

274 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

275 continue 

276 if "cmd" not in resources: 

277 log.warning("No commands found in %s, skipping.", pluginName) 

278 continue 

279 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

280 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

281 return commands 

282 

283 def _getCommands(self) -> defaultdict[str, list[str]]: 

284 """Get the commands offered by daf_butler and plugin packages. 

285 

286 Returns 

287 ------- 

288 commands : `defaultdict` [`str`, `list` [`str`]] 

289 The key is the command name. The value is a list of package(s) that 

290 contains the command. 

291 """ 

292 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

293 

294 @staticmethod 

295 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None: 

296 """If any provided command is offered by more than one package raise an 

297 exception. 

298 

299 Parameters 

300 ---------- 

301 commands : `defaultdict` [`str`, `list` [`str`]] 

302 The key is the command name. The value is a list of package(s) that 

303 contains the command. 

304 

305 Raises 

306 ------ 

307 click.ClickException 

308 Raised if a command is offered by more than one package, with an 

309 error message to be displayed to the user. 

310 """ 

311 

312 msg = "" 

313 for command, packages in commands.items(): 

314 if len(packages) > 1: 

315 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

316 if msg: 

317 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.") 

318 

319 

320class ButlerCLI(LoaderCLI): 

321 

322 localCmdPkg = "lsst.daf.butler.cli.cmd" 

323 

324 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

325 

326 @classmethod 

327 def _funcNameToCmdName(cls, functionName: str) -> str: 

328 # Docstring inherited from base class. 

329 

330 # The "import" command name and "butler_import" function name are 

331 # defined in cli/cmd/commands.py, and if those names are changed they 

332 # must be changed here as well. 

333 # It is expected that there will be very few butler command names that 

334 # need to be changed because of e.g. conflicts with python keywords (as 

335 # is done here and in _cmdNameToFuncName for the 'import' command). If 

336 # this becomes a common need then some way of doing this should be 

337 # invented that is better than hard coding the function names into 

338 # these conversion functions. An extension of the 'cli/resources.yaml' 

339 # file (as is currently used in obs_base) might be a good way to do it. 

340 if functionName == "butler_import": 

341 return "import" 

342 return super()._funcNameToCmdName(functionName) 

343 

344 @classmethod 

345 def _cmdNameToFuncName(cls, commandName: str) -> str: 

346 # Docstring inherited from base class. 

347 if commandName == "import": 

348 return "butler_import" 

349 return super()._cmdNameToFuncName(commandName) 

350 

351 

352@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

353@log_level_option() 

354@long_log_option() 

355@log_file_option() 

356@log_tty_option() 

357@log_label_option() 

358@ClickProgressHandler.option 

359def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None: 

360 # log_level is handled by get_command and list_commands, and is called in 

361 # one of those functions before this is called. long_log is handled by 

362 # setup_logging. 

363 pass 

364 

365 

366def main() -> click.Command: 

367 return cli()