Coverage for python/lsst/daf/butler/cli/butler.py: 34%
132 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 14:18 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-02 14:18 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "LoaderCLI",
25 "ButlerCLI",
26 "cli",
27 "main",
28)
31import abc
32import functools
33import logging
34import os
35import traceback
36import types
37from collections import defaultdict
38from typing import Any
40import click
41import yaml
42from lsst.utils import doImport
44from .cliLog import CliLog
45from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
46from .progress import ClickProgressHandler
48log = logging.getLogger(__name__)
51@functools.lru_cache
52def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command:
53 """Import a plugin that contains Click commands.
55 Parameters
56 ----------
57 pluginName : `str`
58 An importable module whose __all__ parameter contains the commands
59 that can be called.
61 Returns
62 -------
63 An imported module or None
64 The imported module, or None if the module could not be imported.
66 Notes
67 -----
68 A cache is used in order to prevent repeated reports of failure
69 to import a module that can be triggered by ``butler --help``.
70 """
71 try:
72 return doImport(pluginName)
73 except Exception as err:
74 log.warning("Could not import plugin from %s, skipping.", pluginName)
75 log.debug(
76 "Plugin import exception: %s\nTraceback:\n%s",
77 err,
78 "".join(traceback.format_tb(err.__traceback__)),
79 )
80 return None
83class LoaderCLI(click.MultiCommand, abc.ABC):
84 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
85 subcommands at runtime."""
87 def __init__(self, *args: Any, **kwargs: Any) -> None:
88 super().__init__(*args, **kwargs)
90 @property
91 @abc.abstractmethod
92 def localCmdPkg(self) -> str:
93 """localCmdPkg identifies the location of the commands that are in this
94 package. `getLocalCommands` assumes that the commands can be found in
95 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
96 should be overridden.
98 Returns
99 -------
100 package : `str`
101 The fully qualified location of this package.
102 """
103 raise NotImplementedError()
105 def getLocalCommands(self) -> defaultdict[str, list[str]]:
106 """Get the commands offered by the local package. This assumes that the
107 commands can be found in `localCmdPkg.__all__`, if this is not the case
108 then this function should be overridden.
110 Returns
111 -------
112 commands : `defaultdict` [`str`, `list` [`str`]]
113 The key is the command name. The value is a list of package(s) that
114 contains the command.
115 """
116 commandsLocation = _importPlugin(self.localCmdPkg)
117 if commandsLocation is None:
118 # _importPlugins logs an error, don't need to do it again here.
119 return defaultdict(list)
120 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}"
121 return defaultdict(
122 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
123 )
125 def list_commands(self, ctx: click.Context) -> list[str]:
126 """Used by Click to get all the commands that can be called by the
127 butler command, it is used to generate the --help output.
129 Parameters
130 ----------
131 ctx : `click.Context`
132 The current Click context.
134 Returns
135 -------
136 commands : `list` [`str`]
137 The names of the commands that can be called by the butler command.
138 """
139 self._setupLogging(ctx)
140 commands = self._getCommands()
141 self._raiseIfDuplicateCommands(commands)
142 return sorted(commands)
144 def get_command(self, ctx: click.Context, name: str) -> click.Command | None:
145 """Used by Click to get a single command for execution.
147 Parameters
148 ----------
149 ctx : `click.Context`
150 The current Click context.
151 name : `str`
152 The name of the command to return.
154 Returns
155 -------
156 command : `click.Command`
157 A Command that wraps a callable command function.
158 """
159 self._setupLogging(ctx)
160 commands = self._getCommands()
161 if name not in commands:
162 return None
163 self._raiseIfDuplicateCommands(commands)
164 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name)
165 # The click.command decorator returns an instance of a class, which
166 # is something that doImport is not expecting. We add it in as an
167 # option here to appease mypy.
168 plugin = _importPlugin(module_str)
169 if not plugin:
170 return None
171 if not isinstance(plugin, click.Command):
172 raise RuntimeError(
173 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}"
174 )
175 return plugin
177 def _setupLogging(self, ctx: click.Context | None) -> None:
178 """Init the logging system and config it for the command.
180 Subcommands may further configure the log settings."""
181 if isinstance(ctx, click.Context):
182 CliLog.initLog(
183 longlog=ctx.params.get(long_log_option.name(), False),
184 log_tty=ctx.params.get(log_tty_option.name(), True),
185 log_file=ctx.params.get(log_file_option.name(), ()),
186 log_label=ctx.params.get(log_label_option.name(), ()),
187 )
188 if log_level_option.name() in ctx.params:
189 CliLog.setLogLevels(ctx.params[log_level_option.name()])
190 else:
191 # This works around a bug in sphinx-click, where it passes in the
192 # click.MultiCommand instead of the context.
193 # https://github.com/click-contrib/sphinx-click/issues/70
194 CliLog.initLog(longlog=False)
195 logging.debug(
196 "The passed-in context was not a click.Context, could not determine --long-log or "
197 "--log-level values."
198 )
200 @classmethod
201 def getPluginList(cls) -> list[str]:
202 """Get the list of importable yaml files that contain cli data for this
203 command.
205 Returns
206 -------
207 `list` [`str`]
208 The list of files that contain yaml data about a cli plugin.
209 """
210 if not hasattr(cls, "pluginEnvVar"):
211 return []
212 pluginModules = os.environ.get(cls.pluginEnvVar)
213 if pluginModules:
214 return [p for p in pluginModules.split(":") if p != ""]
215 return []
217 @classmethod
218 def _funcNameToCmdName(cls, functionName: str) -> str:
219 """Convert function name to the butler command name: change
220 underscores, (used in functions) to dashes (used in commands), and
221 change local-package command names that conflict with python keywords
222 to a legal function name.
223 """
224 return functionName.replace("_", "-")
226 @classmethod
227 def _cmdNameToFuncName(cls, commandName: str) -> str:
228 """Convert butler command name to function name: change dashes (used in
229 commands) to underscores (used in functions), and for local-package
230 commands names that conflict with python keywords, change the local,
231 legal, function name to the command name."""
232 return commandName.replace("-", "_")
234 @staticmethod
235 def _mergeCommandLists(
236 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]]
237 ) -> defaultdict[str, list[str]]:
238 """Combine two dicts whose keys are strings (command name) and values
239 are list of string (the package(s) that provide the named command).
241 Parameters
242 ----------
243 a : `defaultdict` [`str`, `list` [`str`]]
244 The key is the command name. The value is a list of package(s) that
245 contains the command.
246 b : (same as a)
248 Returns
249 -------
250 commands : `defaultdict` [`str`: [`str`]]
251 For convenience, returns a extended with b. ('a' is modified in
252 place.)
253 """
254 for key, val in b.items():
255 a[key].extend(val)
256 return a
258 @classmethod
259 def _getPluginCommands(cls) -> defaultdict[str, list[str]]:
260 """Get the commands offered by plugin packages.
262 Returns
263 -------
264 commands : `defaultdict` [`str`, `list` [`str`]]
265 The key is the command name. The value is a list of package(s) that
266 contains the command.
267 """
268 commands: defaultdict[str, list[str]] = defaultdict(list)
269 for pluginName in cls.getPluginList():
270 try:
271 with open(pluginName, "r") as resourceFile:
272 resources = defaultdict(list, yaml.safe_load(resourceFile))
273 except Exception as err:
274 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
275 continue
276 if "cmd" not in resources:
277 log.warning("No commands found in %s, skipping.", pluginName)
278 continue
279 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
280 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
281 return commands
283 def _getCommands(self) -> defaultdict[str, list[str]]:
284 """Get the commands offered by daf_butler and plugin packages.
286 Returns
287 -------
288 commands : `defaultdict` [`str`, `list` [`str`]]
289 The key is the command name. The value is a list of package(s) that
290 contains the command.
291 """
292 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
294 @staticmethod
295 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None:
296 """If any provided command is offered by more than one package raise an
297 exception.
299 Parameters
300 ----------
301 commands : `defaultdict` [`str`, `list` [`str`]]
302 The key is the command name. The value is a list of package(s) that
303 contains the command.
305 Raises
306 ------
307 click.ClickException
308 Raised if a command is offered by more than one package, with an
309 error message to be displayed to the user.
310 """
312 msg = ""
313 for command, packages in commands.items():
314 if len(packages) > 1:
315 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
316 if msg:
317 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.")
320class ButlerCLI(LoaderCLI):
321 localCmdPkg = "lsst.daf.butler.cli.cmd"
323 pluginEnvVar = "DAF_BUTLER_PLUGINS"
325 @classmethod
326 def _funcNameToCmdName(cls, functionName: str) -> str:
327 # Docstring inherited from base class.
329 # The "import" command name and "butler_import" function name are
330 # defined in cli/cmd/commands.py, and if those names are changed they
331 # must be changed here as well.
332 # It is expected that there will be very few butler command names that
333 # need to be changed because of e.g. conflicts with python keywords (as
334 # is done here and in _cmdNameToFuncName for the 'import' command). If
335 # this becomes a common need then some way of doing this should be
336 # invented that is better than hard coding the function names into
337 # these conversion functions. An extension of the 'cli/resources.yaml'
338 # file (as is currently used in obs_base) might be a good way to do it.
339 if functionName == "butler_import":
340 return "import"
341 return super()._funcNameToCmdName(functionName)
343 @classmethod
344 def _cmdNameToFuncName(cls, commandName: str) -> str:
345 # Docstring inherited from base class.
346 if commandName == "import":
347 return "butler_import"
348 return super()._cmdNameToFuncName(commandName)
351@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
352@log_level_option()
353@long_log_option()
354@log_file_option()
355@log_tty_option()
356@log_label_option()
357@ClickProgressHandler.option
358def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None:
359 # log_level is handled by get_command and list_commands, and is called in
360 # one of those functions before this is called. long_log is handled by
361 # setup_logging.
362 pass
365def main() -> click.Command:
366 return cli()