Coverage for python/lsst/daf/butler/cli/butler.py: 46%

131 statements  

« prev     ^ index     » next       coverage.py v7.3.1, created at 2023-10-02 08:00 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "LoaderCLI", 

31 "ButlerCLI", 

32 "cli", 

33 "main", 

34) 

35 

36 

37import abc 

38import functools 

39import logging 

40import os 

41import traceback 

42import types 

43from collections import defaultdict 

44from typing import Any 

45 

46import click 

47import yaml 

48from lsst.utils import doImport 

49 

50from .cliLog import CliLog 

51from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

52from .progress import ClickProgressHandler 

53 

54log = logging.getLogger(__name__) 

55 

56 

57@functools.lru_cache 

58def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command: 

59 """Import a plugin that contains Click commands. 

60 

61 Parameters 

62 ---------- 

63 pluginName : `str` 

64 An importable module whose __all__ parameter contains the commands 

65 that can be called. 

66 

67 Returns 

68 ------- 

69 An imported module or None 

70 The imported module, or None if the module could not be imported. 

71 

72 Notes 

73 ----- 

74 A cache is used in order to prevent repeated reports of failure 

75 to import a module that can be triggered by ``butler --help``. 

76 """ 

77 try: 

78 return doImport(pluginName) 

79 except Exception as err: 

80 log.warning("Could not import plugin from %s, skipping.", pluginName) 

81 log.debug( 

82 "Plugin import exception: %s\nTraceback:\n%s", 

83 err, 

84 "".join(traceback.format_tb(err.__traceback__)), 

85 ) 

86 return None 

87 

88 

89class LoaderCLI(click.MultiCommand, abc.ABC): 

90 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

91 subcommands at runtime. 

92 """ 

93 

94 def __init__(self, *args: Any, **kwargs: Any) -> None: 

95 super().__init__(*args, **kwargs) 

96 

97 @property 

98 @abc.abstractmethod 

99 def localCmdPkg(self) -> str: 

100 """Identifies the location of the commands that are in this 

101 package. 

102 

103 `getLocalCommands` assumes that the commands can be found in 

104 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

105 should be overridden. 

106 

107 Returns 

108 ------- 

109 package : `str` 

110 The fully qualified location of this package. 

111 """ 

112 raise NotImplementedError() 

113 

114 def getLocalCommands(self) -> defaultdict[str, list[str]]: 

115 """Get the commands offered by the local package. This assumes that the 

116 commands can be found in `localCmdPkg.__all__`, if this is not the case 

117 then this function should be overridden. 

118 

119 Returns 

120 ------- 

121 commands : `defaultdict` [`str`, `list` [`str`]] 

122 The key is the command name. The value is a list of package(s) that 

123 contains the command. 

124 """ 

125 commandsLocation = _importPlugin(self.localCmdPkg) 

126 if commandsLocation is None: 

127 # _importPlugins logs an error, don't need to do it again here. 

128 return defaultdict(list) 

129 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}" 

130 return defaultdict( 

131 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

132 ) 

133 

134 def list_commands(self, ctx: click.Context) -> list[str]: 

135 """Get all the commands that can be called by the 

136 butler command, it is used to generate the --help output. 

137 

138 Used by Click. 

139 

140 Parameters 

141 ---------- 

142 ctx : `click.Context` 

143 The current Click context. 

144 

145 Returns 

146 ------- 

147 commands : `list` [`str`] 

148 The names of the commands that can be called by the butler command. 

149 """ 

150 self._setupLogging(ctx) 

151 commands = self._getCommands() 

152 self._raiseIfDuplicateCommands(commands) 

153 return sorted(commands) 

154 

155 def get_command(self, ctx: click.Context, name: str) -> click.Command | None: 

156 """Get a single command for execution. 

157 

158 Used by Click. 

159 

160 Parameters 

161 ---------- 

162 ctx : `click.Context` 

163 The current Click context. 

164 name : `str` 

165 The name of the command to return. 

166 

167 Returns 

168 ------- 

169 command : `click.Command` 

170 A Command that wraps a callable command function. 

171 """ 

172 self._setupLogging(ctx) 

173 commands = self._getCommands() 

174 if name not in commands: 

175 return None 

176 self._raiseIfDuplicateCommands(commands) 

177 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name) 

178 # The click.command decorator returns an instance of a class, which 

179 # is something that doImport is not expecting. We add it in as an 

180 # option here to appease mypy. 

181 plugin = _importPlugin(module_str) 

182 if not plugin: 

183 return None 

184 if not isinstance(plugin, click.Command): 

185 raise RuntimeError( 

186 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}" 

187 ) 

188 return plugin 

189 

190 def _setupLogging(self, ctx: click.Context | None) -> None: 

191 """Init the logging system and config it for the command. 

192 

193 Subcommands may further configure the log settings. 

194 """ 

195 if isinstance(ctx, click.Context): 

196 CliLog.initLog( 

197 longlog=ctx.params.get(long_log_option.name(), False), 

198 log_tty=ctx.params.get(log_tty_option.name(), True), 

199 log_file=ctx.params.get(log_file_option.name(), ()), 

200 log_label=ctx.params.get(log_label_option.name(), ()), 

201 ) 

202 if log_level_option.name() in ctx.params: 

203 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

204 else: 

205 # This works around a bug in sphinx-click, where it passes in the 

206 # click.MultiCommand instead of the context. 

207 # https://github.com/click-contrib/sphinx-click/issues/70 

208 CliLog.initLog(longlog=False) 

209 logging.debug( 

210 "The passed-in context was not a click.Context, could not determine --long-log or " 

211 "--log-level values." 

212 ) 

213 

214 @classmethod 

215 def getPluginList(cls) -> list[str]: 

216 """Get the list of importable yaml files that contain cli data for this 

217 command. 

218 

219 Returns 

220 ------- 

221 `list` [`str`] 

222 The list of files that contain yaml data about a cli plugin. 

223 """ 

224 if not hasattr(cls, "pluginEnvVar"): 

225 return [] 

226 pluginModules = os.environ.get(cls.pluginEnvVar) 

227 if pluginModules: 

228 return [p for p in pluginModules.split(":") if p != ""] 

229 return [] 

230 

231 @classmethod 

232 def _funcNameToCmdName(cls, functionName: str) -> str: 

233 """Convert function name to the butler command name: change 

234 underscores, (used in functions) to dashes (used in commands), and 

235 change local-package command names that conflict with python keywords 

236 to a legal function name. 

237 """ 

238 return functionName.replace("_", "-") 

239 

240 @classmethod 

241 def _cmdNameToFuncName(cls, commandName: str) -> str: 

242 """Convert butler command name to function name: change dashes (used in 

243 commands) to underscores (used in functions), and for local-package 

244 commands names that conflict with python keywords, change the local, 

245 legal, function name to the command name. 

246 """ 

247 return commandName.replace("-", "_") 

248 

249 @staticmethod 

250 def _mergeCommandLists( 

251 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]] 

252 ) -> defaultdict[str, list[str]]: 

253 """Combine two dicts whose keys are strings (command name) and values 

254 are list of string (the package(s) that provide the named command). 

255 

256 Parameters 

257 ---------- 

258 a : `defaultdict` [`str`, `list` [`str`]] 

259 The key is the command name. The value is a list of package(s) that 

260 contains the command. 

261 b : (same as a) 

262 

263 Returns 

264 ------- 

265 commands : `defaultdict` [`str`: [`str`]] 

266 For convenience, returns a extended with b. ('a' is modified in 

267 place.) 

268 """ 

269 for key, val in b.items(): 

270 a[key].extend(val) 

271 return a 

272 

273 @classmethod 

274 def _getPluginCommands(cls) -> defaultdict[str, list[str]]: 

275 """Get the commands offered by plugin packages. 

276 

277 Returns 

278 ------- 

279 commands : `defaultdict` [`str`, `list` [`str`]] 

280 The key is the command name. The value is a list of package(s) that 

281 contains the command. 

282 """ 

283 commands: defaultdict[str, list[str]] = defaultdict(list) 

284 for pluginName in cls.getPluginList(): 

285 try: 

286 with open(pluginName) as resourceFile: 

287 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

288 except Exception as err: 

289 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

290 continue 

291 if "cmd" not in resources: 

292 log.warning("No commands found in %s, skipping.", pluginName) 

293 continue 

294 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

295 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

296 return commands 

297 

298 def _getCommands(self) -> defaultdict[str, list[str]]: 

299 """Get the commands offered by daf_butler and plugin packages. 

300 

301 Returns 

302 ------- 

303 commands : `defaultdict` [`str`, `list` [`str`]] 

304 The key is the command name. The value is a list of package(s) that 

305 contains the command. 

306 """ 

307 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

308 

309 @staticmethod 

310 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None: 

311 """If any provided command is offered by more than one package raise an 

312 exception. 

313 

314 Parameters 

315 ---------- 

316 commands : `defaultdict` [`str`, `list` [`str`]] 

317 The key is the command name. The value is a list of package(s) that 

318 contains the command. 

319 

320 Raises 

321 ------ 

322 click.ClickException 

323 Raised if a command is offered by more than one package, with an 

324 error message to be displayed to the user. 

325 """ 

326 msg = "" 

327 for command, packages in commands.items(): 

328 if len(packages) > 1: 

329 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

330 if msg: 

331 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.") 

332 

333 

334class ButlerCLI(LoaderCLI): 

335 """Specialized command loader implementing the ``butler`` command.""" 

336 

337 localCmdPkg = "lsst.daf.butler.cli.cmd" 

338 

339 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

340 

341 @classmethod 

342 def _funcNameToCmdName(cls, functionName: str) -> str: 

343 # Docstring inherited from base class. 

344 

345 # The "import" command name and "butler_import" function name are 

346 # defined in cli/cmd/commands.py, and if those names are changed they 

347 # must be changed here as well. 

348 # It is expected that there will be very few butler command names that 

349 # need to be changed because of e.g. conflicts with python keywords (as 

350 # is done here and in _cmdNameToFuncName for the 'import' command). If 

351 # this becomes a common need then some way of doing this should be 

352 # invented that is better than hard coding the function names into 

353 # these conversion functions. An extension of the 'cli/resources.yaml' 

354 # file (as is currently used in obs_base) might be a good way to do it. 

355 if functionName == "butler_import": 

356 return "import" 

357 return super()._funcNameToCmdName(functionName) 

358 

359 @classmethod 

360 def _cmdNameToFuncName(cls, commandName: str) -> str: 

361 # Docstring inherited from base class. 

362 if commandName == "import": 

363 return "butler_import" 

364 return super()._cmdNameToFuncName(commandName) 

365 

366 

367@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

368@log_level_option() 

369@long_log_option() 

370@log_file_option() 

371@log_tty_option() 

372@log_label_option() 

373@ClickProgressHandler.option 

374def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None: 

375 """Command line interface for butler. 

376 

377 log_level is handled by get_command and list_commands, and is called in 

378 one of those functions before this is called. long_log is handled by 

379 setup_logging. 

380 """ 

381 pass 

382 

383 

384def main() -> click.Command: 

385 """Return main entry point for command-line.""" 

386 return cli()