Coverage for python/lsst/daf/butler/cli/butler.py: 46%

131 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-14 19:21 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "LoaderCLI", 

25 "ButlerCLI", 

26 "cli", 

27 "main", 

28) 

29 

30 

31import abc 

32import functools 

33import logging 

34import os 

35import traceback 

36import types 

37from collections import defaultdict 

38from typing import Any 

39 

40import click 

41import yaml 

42from lsst.utils import doImport 

43 

44from .cliLog import CliLog 

45from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

46from .progress import ClickProgressHandler 

47 

48log = logging.getLogger(__name__) 

49 

50 

51@functools.lru_cache 

52def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command: 

53 """Import a plugin that contains Click commands. 

54 

55 Parameters 

56 ---------- 

57 pluginName : `str` 

58 An importable module whose __all__ parameter contains the commands 

59 that can be called. 

60 

61 Returns 

62 ------- 

63 An imported module or None 

64 The imported module, or None if the module could not be imported. 

65 

66 Notes 

67 ----- 

68 A cache is used in order to prevent repeated reports of failure 

69 to import a module that can be triggered by ``butler --help``. 

70 """ 

71 try: 

72 return doImport(pluginName) 

73 except Exception as err: 

74 log.warning("Could not import plugin from %s, skipping.", pluginName) 

75 log.debug( 

76 "Plugin import exception: %s\nTraceback:\n%s", 

77 err, 

78 "".join(traceback.format_tb(err.__traceback__)), 

79 ) 

80 return None 

81 

82 

83class LoaderCLI(click.MultiCommand, abc.ABC): 

84 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

85 subcommands at runtime. 

86 """ 

87 

88 def __init__(self, *args: Any, **kwargs: Any) -> None: 

89 super().__init__(*args, **kwargs) 

90 

91 @property 

92 @abc.abstractmethod 

93 def localCmdPkg(self) -> str: 

94 """Identifies the location of the commands that are in this 

95 package. 

96 

97 `getLocalCommands` assumes that the commands can be found in 

98 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

99 should be overridden. 

100 

101 Returns 

102 ------- 

103 package : `str` 

104 The fully qualified location of this package. 

105 """ 

106 raise NotImplementedError() 

107 

108 def getLocalCommands(self) -> defaultdict[str, list[str]]: 

109 """Get the commands offered by the local package. This assumes that the 

110 commands can be found in `localCmdPkg.__all__`, if this is not the case 

111 then this function should be overridden. 

112 

113 Returns 

114 ------- 

115 commands : `defaultdict` [`str`, `list` [`str`]] 

116 The key is the command name. The value is a list of package(s) that 

117 contains the command. 

118 """ 

119 commandsLocation = _importPlugin(self.localCmdPkg) 

120 if commandsLocation is None: 

121 # _importPlugins logs an error, don't need to do it again here. 

122 return defaultdict(list) 

123 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}" 

124 return defaultdict( 

125 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

126 ) 

127 

128 def list_commands(self, ctx: click.Context) -> list[str]: 

129 """Get all the commands that can be called by the 

130 butler command, it is used to generate the --help output. 

131 

132 Used by Click. 

133 

134 Parameters 

135 ---------- 

136 ctx : `click.Context` 

137 The current Click context. 

138 

139 Returns 

140 ------- 

141 commands : `list` [`str`] 

142 The names of the commands that can be called by the butler command. 

143 """ 

144 self._setupLogging(ctx) 

145 commands = self._getCommands() 

146 self._raiseIfDuplicateCommands(commands) 

147 return sorted(commands) 

148 

149 def get_command(self, ctx: click.Context, name: str) -> click.Command | None: 

150 """Get a single command for execution. 

151 

152 Used by Click. 

153 

154 Parameters 

155 ---------- 

156 ctx : `click.Context` 

157 The current Click context. 

158 name : `str` 

159 The name of the command to return. 

160 

161 Returns 

162 ------- 

163 command : `click.Command` 

164 A Command that wraps a callable command function. 

165 """ 

166 self._setupLogging(ctx) 

167 commands = self._getCommands() 

168 if name not in commands: 

169 return None 

170 self._raiseIfDuplicateCommands(commands) 

171 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name) 

172 # The click.command decorator returns an instance of a class, which 

173 # is something that doImport is not expecting. We add it in as an 

174 # option here to appease mypy. 

175 plugin = _importPlugin(module_str) 

176 if not plugin: 

177 return None 

178 if not isinstance(plugin, click.Command): 

179 raise RuntimeError( 

180 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}" 

181 ) 

182 return plugin 

183 

184 def _setupLogging(self, ctx: click.Context | None) -> None: 

185 """Init the logging system and config it for the command. 

186 

187 Subcommands may further configure the log settings. 

188 """ 

189 if isinstance(ctx, click.Context): 

190 CliLog.initLog( 

191 longlog=ctx.params.get(long_log_option.name(), False), 

192 log_tty=ctx.params.get(log_tty_option.name(), True), 

193 log_file=ctx.params.get(log_file_option.name(), ()), 

194 log_label=ctx.params.get(log_label_option.name(), ()), 

195 ) 

196 if log_level_option.name() in ctx.params: 

197 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

198 else: 

199 # This works around a bug in sphinx-click, where it passes in the 

200 # click.MultiCommand instead of the context. 

201 # https://github.com/click-contrib/sphinx-click/issues/70 

202 CliLog.initLog(longlog=False) 

203 logging.debug( 

204 "The passed-in context was not a click.Context, could not determine --long-log or " 

205 "--log-level values." 

206 ) 

207 

208 @classmethod 

209 def getPluginList(cls) -> list[str]: 

210 """Get the list of importable yaml files that contain cli data for this 

211 command. 

212 

213 Returns 

214 ------- 

215 `list` [`str`] 

216 The list of files that contain yaml data about a cli plugin. 

217 """ 

218 if not hasattr(cls, "pluginEnvVar"): 

219 return [] 

220 pluginModules = os.environ.get(cls.pluginEnvVar) 

221 if pluginModules: 

222 return [p for p in pluginModules.split(":") if p != ""] 

223 return [] 

224 

225 @classmethod 

226 def _funcNameToCmdName(cls, functionName: str) -> str: 

227 """Convert function name to the butler command name: change 

228 underscores, (used in functions) to dashes (used in commands), and 

229 change local-package command names that conflict with python keywords 

230 to a legal function name. 

231 """ 

232 return functionName.replace("_", "-") 

233 

234 @classmethod 

235 def _cmdNameToFuncName(cls, commandName: str) -> str: 

236 """Convert butler command name to function name: change dashes (used in 

237 commands) to underscores (used in functions), and for local-package 

238 commands names that conflict with python keywords, change the local, 

239 legal, function name to the command name. 

240 """ 

241 return commandName.replace("-", "_") 

242 

243 @staticmethod 

244 def _mergeCommandLists( 

245 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]] 

246 ) -> defaultdict[str, list[str]]: 

247 """Combine two dicts whose keys are strings (command name) and values 

248 are list of string (the package(s) that provide the named command). 

249 

250 Parameters 

251 ---------- 

252 a : `defaultdict` [`str`, `list` [`str`]] 

253 The key is the command name. The value is a list of package(s) that 

254 contains the command. 

255 b : (same as a) 

256 

257 Returns 

258 ------- 

259 commands : `defaultdict` [`str`: [`str`]] 

260 For convenience, returns a extended with b. ('a' is modified in 

261 place.) 

262 """ 

263 for key, val in b.items(): 

264 a[key].extend(val) 

265 return a 

266 

267 @classmethod 

268 def _getPluginCommands(cls) -> defaultdict[str, list[str]]: 

269 """Get the commands offered by plugin packages. 

270 

271 Returns 

272 ------- 

273 commands : `defaultdict` [`str`, `list` [`str`]] 

274 The key is the command name. The value is a list of package(s) that 

275 contains the command. 

276 """ 

277 commands: defaultdict[str, list[str]] = defaultdict(list) 

278 for pluginName in cls.getPluginList(): 

279 try: 

280 with open(pluginName) as resourceFile: 

281 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

282 except Exception as err: 

283 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

284 continue 

285 if "cmd" not in resources: 

286 log.warning("No commands found in %s, skipping.", pluginName) 

287 continue 

288 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

289 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

290 return commands 

291 

292 def _getCommands(self) -> defaultdict[str, list[str]]: 

293 """Get the commands offered by daf_butler and plugin packages. 

294 

295 Returns 

296 ------- 

297 commands : `defaultdict` [`str`, `list` [`str`]] 

298 The key is the command name. The value is a list of package(s) that 

299 contains the command. 

300 """ 

301 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

302 

303 @staticmethod 

304 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None: 

305 """If any provided command is offered by more than one package raise an 

306 exception. 

307 

308 Parameters 

309 ---------- 

310 commands : `defaultdict` [`str`, `list` [`str`]] 

311 The key is the command name. The value is a list of package(s) that 

312 contains the command. 

313 

314 Raises 

315 ------ 

316 click.ClickException 

317 Raised if a command is offered by more than one package, with an 

318 error message to be displayed to the user. 

319 """ 

320 msg = "" 

321 for command, packages in commands.items(): 

322 if len(packages) > 1: 

323 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

324 if msg: 

325 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.") 

326 

327 

328class ButlerCLI(LoaderCLI): 

329 """Specialized command loader implementing the ``butler`` command.""" 

330 

331 localCmdPkg = "lsst.daf.butler.cli.cmd" 

332 

333 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

334 

335 @classmethod 

336 def _funcNameToCmdName(cls, functionName: str) -> str: 

337 # Docstring inherited from base class. 

338 

339 # The "import" command name and "butler_import" function name are 

340 # defined in cli/cmd/commands.py, and if those names are changed they 

341 # must be changed here as well. 

342 # It is expected that there will be very few butler command names that 

343 # need to be changed because of e.g. conflicts with python keywords (as 

344 # is done here and in _cmdNameToFuncName for the 'import' command). If 

345 # this becomes a common need then some way of doing this should be 

346 # invented that is better than hard coding the function names into 

347 # these conversion functions. An extension of the 'cli/resources.yaml' 

348 # file (as is currently used in obs_base) might be a good way to do it. 

349 if functionName == "butler_import": 

350 return "import" 

351 return super()._funcNameToCmdName(functionName) 

352 

353 @classmethod 

354 def _cmdNameToFuncName(cls, commandName: str) -> str: 

355 # Docstring inherited from base class. 

356 if commandName == "import": 

357 return "butler_import" 

358 return super()._cmdNameToFuncName(commandName) 

359 

360 

361@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

362@log_level_option() 

363@long_log_option() 

364@log_file_option() 

365@log_tty_option() 

366@log_label_option() 

367@ClickProgressHandler.option 

368def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None: 

369 """Command line interface for butler. 

370 

371 log_level is handled by get_command and list_commands, and is called in 

372 one of those functions before this is called. long_log is handled by 

373 setup_logging. 

374 """ 

375 pass 

376 

377 

378def main() -> click.Command: 

379 """Return main entry point for command-line.""" 

380 return cli()