Coverage for python/lsst/daf/butler/cli/butler.py: 46%
131 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:53 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-30 02:53 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "LoaderCLI",
31 "ButlerCLI",
32 "cli",
33 "main",
34)
37import abc
38import functools
39import logging
40import os
41import traceback
42import types
43from collections import defaultdict
44from typing import Any
46import click
47import yaml
48from lsst.utils import doImport
50from .cliLog import CliLog
51from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
52from .progress import ClickProgressHandler
54log = logging.getLogger(__name__)
57@functools.lru_cache
58def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command:
59 """Import a plugin that contains Click commands.
61 Parameters
62 ----------
63 pluginName : `str`
64 An importable module whose __all__ parameter contains the commands
65 that can be called.
67 Returns
68 -------
69 An imported module or None
70 The imported module, or None if the module could not be imported.
72 Notes
73 -----
74 A cache is used in order to prevent repeated reports of failure
75 to import a module that can be triggered by ``butler --help``.
76 """
77 try:
78 return doImport(pluginName)
79 except Exception as err:
80 log.warning("Could not import plugin from %s, skipping.", pluginName)
81 log.debug(
82 "Plugin import exception: %s\nTraceback:\n%s",
83 err,
84 "".join(traceback.format_tb(err.__traceback__)),
85 )
86 return None
89class LoaderCLI(click.MultiCommand, abc.ABC):
90 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
91 subcommands at runtime.
93 Parameters
94 ----------
95 *args : `typing.Any`
96 Arguments passed to parent constructor.
97 **kwargs : `typing.Any`
98 Keyword arguments passed to parent constructor.
99 """
101 def __init__(self, *args: Any, **kwargs: Any) -> None:
102 super().__init__(*args, **kwargs)
104 @property
105 @abc.abstractmethod
106 def localCmdPkg(self) -> str:
107 """Identifies the location of the commands that are in this
108 package.
110 `getLocalCommands` assumes that the commands can be found in
111 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
112 should be overridden.
114 Returns
115 -------
116 package : `str`
117 The fully qualified location of this package.
118 """
119 raise NotImplementedError()
121 def getLocalCommands(self) -> defaultdict[str, list[str]]:
122 """Get the commands offered by the local package. This assumes that the
123 commands can be found in `localCmdPkg.__all__`, if this is not the case
124 then this function should be overridden.
126 Returns
127 -------
128 commands : `defaultdict` [`str`, `list` [`str`]]
129 The key is the command name. The value is a list of package(s) that
130 contains the command.
131 """
132 commandsLocation = _importPlugin(self.localCmdPkg)
133 if commandsLocation is None:
134 # _importPlugins logs an error, don't need to do it again here.
135 return defaultdict(list)
136 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}"
137 return defaultdict(
138 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
139 )
141 def list_commands(self, ctx: click.Context) -> list[str]:
142 """Get all the commands that can be called by the
143 butler command, it is used to generate the --help output.
145 Used by Click.
147 Parameters
148 ----------
149 ctx : `click.Context`
150 The current Click context.
152 Returns
153 -------
154 commands : `list` [`str`]
155 The names of the commands that can be called by the butler command.
156 """
157 self._setupLogging(ctx)
158 commands = self._getCommands()
159 self._raiseIfDuplicateCommands(commands)
160 return sorted(commands)
162 def get_command(self, ctx: click.Context, name: str) -> click.Command | None:
163 """Get a single command for execution.
165 Used by Click.
167 Parameters
168 ----------
169 ctx : `click.Context`
170 The current Click context.
171 name : `str`
172 The name of the command to return.
174 Returns
175 -------
176 command : `click.Command`
177 A Command that wraps a callable command function.
178 """
179 self._setupLogging(ctx)
180 commands = self._getCommands()
181 if name not in commands:
182 return None
183 self._raiseIfDuplicateCommands(commands)
184 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name)
185 # The click.command decorator returns an instance of a class, which
186 # is something that doImport is not expecting. We add it in as an
187 # option here to appease mypy.
188 plugin = _importPlugin(module_str)
189 if not plugin:
190 return None
191 if not isinstance(plugin, click.Command):
192 raise RuntimeError(
193 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}"
194 )
195 return plugin
197 def _setupLogging(self, ctx: click.Context | None) -> None:
198 """Init the logging system and config it for the command.
200 Subcommands may further configure the log settings.
201 """
202 if isinstance(ctx, click.Context):
203 CliLog.initLog(
204 longlog=ctx.params.get(long_log_option.name(), False),
205 log_tty=ctx.params.get(log_tty_option.name(), True),
206 log_file=ctx.params.get(log_file_option.name(), ()),
207 log_label=ctx.params.get(log_label_option.name(), ()),
208 )
209 if log_level_option.name() in ctx.params:
210 CliLog.setLogLevels(ctx.params[log_level_option.name()])
211 else:
212 # This works around a bug in sphinx-click, where it passes in the
213 # click.MultiCommand instead of the context.
214 # https://github.com/click-contrib/sphinx-click/issues/70
215 CliLog.initLog(longlog=False)
216 logging.debug(
217 "The passed-in context was not a click.Context, could not determine --long-log or "
218 "--log-level values."
219 )
221 @classmethod
222 def getPluginList(cls) -> list[str]:
223 """Get the list of importable yaml files that contain cli data for this
224 command.
226 Returns
227 -------
228 `list` [`str`]
229 The list of files that contain yaml data about a cli plugin.
230 """
231 if not hasattr(cls, "pluginEnvVar"):
232 return []
233 pluginModules = os.environ.get(cls.pluginEnvVar)
234 if pluginModules:
235 return [p for p in pluginModules.split(":") if p != ""]
236 return []
238 @classmethod
239 def _funcNameToCmdName(cls, functionName: str) -> str:
240 """Convert function name to the butler command name: change
241 underscores, (used in functions) to dashes (used in commands), and
242 change local-package command names that conflict with python keywords
243 to a legal function name.
244 """
245 return functionName.replace("_", "-")
247 @classmethod
248 def _cmdNameToFuncName(cls, commandName: str) -> str:
249 """Convert butler command name to function name: change dashes (used in
250 commands) to underscores (used in functions), and for local-package
251 commands names that conflict with python keywords, change the local,
252 legal, function name to the command name.
253 """
254 return commandName.replace("-", "_")
256 @staticmethod
257 def _mergeCommandLists(
258 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]]
259 ) -> defaultdict[str, list[str]]:
260 """Combine two dicts whose keys are strings (command name) and values
261 are list of string (the package(s) that provide the named command).
263 Parameters
264 ----------
265 a : `defaultdict` [`str`, `list` [`str`]]
266 The key is the command name. The value is a list of package(s) that
267 contains the command.
268 b : (same as a)
270 Returns
271 -------
272 commands : `defaultdict` [`str`: [`str`]]
273 For convenience, returns a extended with b. ('a' is modified in
274 place.)
275 """
276 for key, val in b.items():
277 a[key].extend(val)
278 return a
280 @classmethod
281 def _getPluginCommands(cls) -> defaultdict[str, list[str]]:
282 """Get the commands offered by plugin packages.
284 Returns
285 -------
286 commands : `defaultdict` [`str`, `list` [`str`]]
287 The key is the command name. The value is a list of package(s) that
288 contains the command.
289 """
290 commands: defaultdict[str, list[str]] = defaultdict(list)
291 for pluginName in cls.getPluginList():
292 try:
293 with open(pluginName) as resourceFile:
294 resources = defaultdict(list, yaml.safe_load(resourceFile))
295 except Exception as err:
296 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
297 continue
298 if "cmd" not in resources:
299 log.warning("No commands found in %s, skipping.", pluginName)
300 continue
301 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
302 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
303 return commands
305 def _getCommands(self) -> defaultdict[str, list[str]]:
306 """Get the commands offered by daf_butler and plugin packages.
308 Returns
309 -------
310 commands : `defaultdict` [`str`, `list` [`str`]]
311 The key is the command name. The value is a list of package(s) that
312 contains the command.
313 """
314 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
316 @staticmethod
317 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None:
318 """If any provided command is offered by more than one package raise an
319 exception.
321 Parameters
322 ----------
323 commands : `defaultdict` [`str`, `list` [`str`]]
324 The key is the command name. The value is a list of package(s) that
325 contains the command.
327 Raises
328 ------
329 click.ClickException
330 Raised if a command is offered by more than one package, with an
331 error message to be displayed to the user.
332 """
333 msg = ""
334 for command, packages in commands.items():
335 if len(packages) > 1:
336 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
337 if msg:
338 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.")
341class ButlerCLI(LoaderCLI):
342 """Specialized command loader implementing the ``butler`` command."""
344 localCmdPkg = "lsst.daf.butler.cli.cmd"
346 pluginEnvVar = "DAF_BUTLER_PLUGINS"
348 @classmethod
349 def _funcNameToCmdName(cls, functionName: str) -> str:
350 # Docstring inherited from base class.
352 # The "import" command name and "butler_import" function name are
353 # defined in cli/cmd/commands.py, and if those names are changed they
354 # must be changed here as well.
355 # It is expected that there will be very few butler command names that
356 # need to be changed because of e.g. conflicts with python keywords (as
357 # is done here and in _cmdNameToFuncName for the 'import' command). If
358 # this becomes a common need then some way of doing this should be
359 # invented that is better than hard coding the function names into
360 # these conversion functions. An extension of the 'cli/resources.yaml'
361 # file (as is currently used in obs_base) might be a good way to do it.
362 if functionName == "butler_import":
363 return "import"
364 return super()._funcNameToCmdName(functionName)
366 @classmethod
367 def _cmdNameToFuncName(cls, commandName: str) -> str:
368 # Docstring inherited from base class.
369 if commandName == "import":
370 return "butler_import"
371 return super()._cmdNameToFuncName(commandName)
374@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
375@log_level_option()
376@long_log_option()
377@log_file_option()
378@log_tty_option()
379@log_label_option()
380@ClickProgressHandler.option
381def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None:
382 """Command line interface for butler.
384 Parameters
385 ----------
386 log_level : `str`
387 The log level to use by default. ``log_level`` is handled by
388 ``get_command`` and ``list_commands``, and is called in
389 one of those functions before this is called.
390 long_log : `bool`
391 Enable extended log output. ``long_log`` is handled by
392 ``setup_logging``.
393 log_file : `str`
394 The log file name.
395 log_tty : `bool`
396 Whether to send logs to standard output.
397 log_label : `str`
398 Log labels.
399 progress : `bool`
400 Whether to use progress bar.
401 """
402 pass
405def main() -> click.Command:
406 """Return main entry point for command-line."""
407 return cli()