Coverage for python/lsst/daf/butler/cli/butler.py: 46%

131 statements  

« prev     ^ index     » next       coverage.py v7.4.4, created at 2024-03-30 09:59 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "LoaderCLI", 

31 "ButlerCLI", 

32 "cli", 

33 "main", 

34) 

35 

36 

37import abc 

38import functools 

39import logging 

40import os 

41import traceback 

42import types 

43from collections import defaultdict 

44from typing import Any 

45 

46import click 

47import yaml 

48from lsst.utils import doImport 

49 

50from .cliLog import CliLog 

51from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

52from .progress import ClickProgressHandler 

53 

54log = logging.getLogger(__name__) 

55 

56 

57@functools.lru_cache 

58def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command: 

59 """Import a plugin that contains Click commands. 

60 

61 Parameters 

62 ---------- 

63 pluginName : `str` 

64 An importable module whose __all__ parameter contains the commands 

65 that can be called. 

66 

67 Returns 

68 ------- 

69 An imported module or None 

70 The imported module, or None if the module could not be imported. 

71 

72 Notes 

73 ----- 

74 A cache is used in order to prevent repeated reports of failure 

75 to import a module that can be triggered by ``butler --help``. 

76 """ 

77 try: 

78 return doImport(pluginName) 

79 except Exception as err: 

80 log.warning("Could not import plugin from %s, skipping.", pluginName) 

81 log.debug( 

82 "Plugin import exception: %s\nTraceback:\n%s", 

83 err, 

84 "".join(traceback.format_tb(err.__traceback__)), 

85 ) 

86 return None 

87 

88 

89class LoaderCLI(click.MultiCommand, abc.ABC): 

90 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

91 subcommands at runtime. 

92 

93 Parameters 

94 ---------- 

95 *args : `typing.Any` 

96 Arguments passed to parent constructor. 

97 **kwargs : `typing.Any` 

98 Keyword arguments passed to parent constructor. 

99 """ 

100 

101 def __init__(self, *args: Any, **kwargs: Any) -> None: 

102 super().__init__(*args, **kwargs) 

103 

104 @property 

105 @abc.abstractmethod 

106 def localCmdPkg(self) -> str: 

107 """Identifies the location of the commands that are in this 

108 package. 

109 

110 `getLocalCommands` assumes that the commands can be found in 

111 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

112 should be overridden. 

113 

114 Returns 

115 ------- 

116 package : `str` 

117 The fully qualified location of this package. 

118 """ 

119 raise NotImplementedError() 

120 

121 def getLocalCommands(self) -> defaultdict[str, list[str]]: 

122 """Get the commands offered by the local package. This assumes that the 

123 commands can be found in `localCmdPkg.__all__`, if this is not the case 

124 then this function should be overridden. 

125 

126 Returns 

127 ------- 

128 commands : `defaultdict` [`str`, `list` [`str`]] 

129 The key is the command name. The value is a list of package(s) that 

130 contains the command. 

131 """ 

132 commandsLocation = _importPlugin(self.localCmdPkg) 

133 if commandsLocation is None: 

134 # _importPlugins logs an error, don't need to do it again here. 

135 return defaultdict(list) 

136 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}" 

137 return defaultdict( 

138 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

139 ) 

140 

141 def list_commands(self, ctx: click.Context) -> list[str]: 

142 """Get all the commands that can be called by the 

143 butler command, it is used to generate the --help output. 

144 

145 Used by Click. 

146 

147 Parameters 

148 ---------- 

149 ctx : `click.Context` 

150 The current Click context. 

151 

152 Returns 

153 ------- 

154 commands : `list` [`str`] 

155 The names of the commands that can be called by the butler command. 

156 """ 

157 self._setupLogging(ctx) 

158 commands = self._getCommands() 

159 self._raiseIfDuplicateCommands(commands) 

160 return sorted(commands) 

161 

162 def get_command(self, ctx: click.Context, name: str) -> click.Command | None: 

163 """Get a single command for execution. 

164 

165 Used by Click. 

166 

167 Parameters 

168 ---------- 

169 ctx : `click.Context` 

170 The current Click context. 

171 name : `str` 

172 The name of the command to return. 

173 

174 Returns 

175 ------- 

176 command : `click.Command` 

177 A Command that wraps a callable command function. 

178 """ 

179 self._setupLogging(ctx) 

180 commands = self._getCommands() 

181 if name not in commands: 

182 return None 

183 self._raiseIfDuplicateCommands(commands) 

184 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name) 

185 # The click.command decorator returns an instance of a class, which 

186 # is something that doImport is not expecting. We add it in as an 

187 # option here to appease mypy. 

188 plugin = _importPlugin(module_str) 

189 if not plugin: 

190 return None 

191 if not isinstance(plugin, click.Command): 

192 raise RuntimeError( 

193 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}" 

194 ) 

195 return plugin 

196 

197 def _setupLogging(self, ctx: click.Context | None) -> None: 

198 """Init the logging system and config it for the command. 

199 

200 Subcommands may further configure the log settings. 

201 """ 

202 if isinstance(ctx, click.Context): 

203 CliLog.initLog( 

204 longlog=ctx.params.get(long_log_option.name(), False), 

205 log_tty=ctx.params.get(log_tty_option.name(), True), 

206 log_file=ctx.params.get(log_file_option.name(), ()), 

207 log_label=ctx.params.get(log_label_option.name(), ()), 

208 ) 

209 if log_level_option.name() in ctx.params: 

210 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

211 else: 

212 # This works around a bug in sphinx-click, where it passes in the 

213 # click.MultiCommand instead of the context. 

214 # https://github.com/click-contrib/sphinx-click/issues/70 

215 CliLog.initLog(longlog=False) 

216 logging.debug( 

217 "The passed-in context was not a click.Context, could not determine --long-log or " 

218 "--log-level values." 

219 ) 

220 

221 @classmethod 

222 def getPluginList(cls) -> list[str]: 

223 """Get the list of importable yaml files that contain cli data for this 

224 command. 

225 

226 Returns 

227 ------- 

228 `list` [`str`] 

229 The list of files that contain yaml data about a cli plugin. 

230 """ 

231 if not hasattr(cls, "pluginEnvVar"): 

232 return [] 

233 pluginModules = os.environ.get(cls.pluginEnvVar) 

234 if pluginModules: 

235 return [p for p in pluginModules.split(":") if p != ""] 

236 return [] 

237 

238 @classmethod 

239 def _funcNameToCmdName(cls, functionName: str) -> str: 

240 """Convert function name to the butler command name: change 

241 underscores, (used in functions) to dashes (used in commands), and 

242 change local-package command names that conflict with python keywords 

243 to a legal function name. 

244 """ 

245 return functionName.replace("_", "-") 

246 

247 @classmethod 

248 def _cmdNameToFuncName(cls, commandName: str) -> str: 

249 """Convert butler command name to function name: change dashes (used in 

250 commands) to underscores (used in functions), and for local-package 

251 commands names that conflict with python keywords, change the local, 

252 legal, function name to the command name. 

253 """ 

254 return commandName.replace("-", "_") 

255 

256 @staticmethod 

257 def _mergeCommandLists( 

258 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]] 

259 ) -> defaultdict[str, list[str]]: 

260 """Combine two dicts whose keys are strings (command name) and values 

261 are list of string (the package(s) that provide the named command). 

262 

263 Parameters 

264 ---------- 

265 a : `defaultdict` [`str`, `list` [`str`]] 

266 The key is the command name. The value is a list of package(s) that 

267 contains the command. 

268 b : (same as a) 

269 

270 Returns 

271 ------- 

272 commands : `defaultdict` [`str`: [`str`]] 

273 For convenience, returns a extended with b. ('a' is modified in 

274 place.) 

275 """ 

276 for key, val in b.items(): 

277 a[key].extend(val) 

278 return a 

279 

280 @classmethod 

281 def _getPluginCommands(cls) -> defaultdict[str, list[str]]: 

282 """Get the commands offered by plugin packages. 

283 

284 Returns 

285 ------- 

286 commands : `defaultdict` [`str`, `list` [`str`]] 

287 The key is the command name. The value is a list of package(s) that 

288 contains the command. 

289 """ 

290 commands: defaultdict[str, list[str]] = defaultdict(list) 

291 for pluginName in cls.getPluginList(): 

292 try: 

293 with open(pluginName) as resourceFile: 

294 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

295 except Exception as err: 

296 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

297 continue 

298 if "cmd" not in resources: 

299 log.warning("No commands found in %s, skipping.", pluginName) 

300 continue 

301 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

302 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

303 return commands 

304 

305 def _getCommands(self) -> defaultdict[str, list[str]]: 

306 """Get the commands offered by daf_butler and plugin packages. 

307 

308 Returns 

309 ------- 

310 commands : `defaultdict` [`str`, `list` [`str`]] 

311 The key is the command name. The value is a list of package(s) that 

312 contains the command. 

313 """ 

314 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

315 

316 @staticmethod 

317 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None: 

318 """If any provided command is offered by more than one package raise an 

319 exception. 

320 

321 Parameters 

322 ---------- 

323 commands : `defaultdict` [`str`, `list` [`str`]] 

324 The key is the command name. The value is a list of package(s) that 

325 contains the command. 

326 

327 Raises 

328 ------ 

329 click.ClickException 

330 Raised if a command is offered by more than one package, with an 

331 error message to be displayed to the user. 

332 """ 

333 msg = "" 

334 for command, packages in commands.items(): 

335 if len(packages) > 1: 

336 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

337 if msg: 

338 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.") 

339 

340 

341class ButlerCLI(LoaderCLI): 

342 """Specialized command loader implementing the ``butler`` command.""" 

343 

344 localCmdPkg = "lsst.daf.butler.cli.cmd" 

345 

346 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

347 

348 @classmethod 

349 def _funcNameToCmdName(cls, functionName: str) -> str: 

350 # Docstring inherited from base class. 

351 

352 # The "import" command name and "butler_import" function name are 

353 # defined in cli/cmd/commands.py, and if those names are changed they 

354 # must be changed here as well. 

355 # It is expected that there will be very few butler command names that 

356 # need to be changed because of e.g. conflicts with python keywords (as 

357 # is done here and in _cmdNameToFuncName for the 'import' command). If 

358 # this becomes a common need then some way of doing this should be 

359 # invented that is better than hard coding the function names into 

360 # these conversion functions. An extension of the 'cli/resources.yaml' 

361 # file (as is currently used in obs_base) might be a good way to do it. 

362 if functionName == "butler_import": 

363 return "import" 

364 return super()._funcNameToCmdName(functionName) 

365 

366 @classmethod 

367 def _cmdNameToFuncName(cls, commandName: str) -> str: 

368 # Docstring inherited from base class. 

369 if commandName == "import": 

370 return "butler_import" 

371 return super()._cmdNameToFuncName(commandName) 

372 

373 

374@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

375@log_level_option() 

376@long_log_option() 

377@log_file_option() 

378@log_tty_option() 

379@log_label_option() 

380@ClickProgressHandler.option 

381def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None: 

382 """Command line interface for butler. 

383 

384 Parameters 

385 ---------- 

386 log_level : `str` 

387 The log level to use by default. ``log_level`` is handled by 

388 ``get_command`` and ``list_commands``, and is called in 

389 one of those functions before this is called. 

390 long_log : `bool` 

391 Enable extended log output. ``long_log`` is handled by 

392 ``setup_logging``. 

393 log_file : `str` 

394 The log file name. 

395 log_tty : `bool` 

396 Whether to send logs to standard output. 

397 log_label : `str` 

398 Log labels. 

399 progress : `bool` 

400 Whether to use progress bar. 

401 """ 

402 pass 

403 

404 

405def main() -> click.Command: 

406 """Return main entry point for command-line.""" 

407 return cli()