Coverage for python/lsst/daf/butler/cli/butler.py: 46%
131 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-12 09:20 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-08-12 09:20 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "LoaderCLI",
25 "ButlerCLI",
26 "cli",
27 "main",
28)
31import abc
32import functools
33import logging
34import os
35import traceback
36import types
37from collections import defaultdict
38from typing import Any
40import click
41import yaml
42from lsst.utils import doImport
44from .cliLog import CliLog
45from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
46from .progress import ClickProgressHandler
48log = logging.getLogger(__name__)
51@functools.lru_cache
52def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command:
53 """Import a plugin that contains Click commands.
55 Parameters
56 ----------
57 pluginName : `str`
58 An importable module whose __all__ parameter contains the commands
59 that can be called.
61 Returns
62 -------
63 An imported module or None
64 The imported module, or None if the module could not be imported.
66 Notes
67 -----
68 A cache is used in order to prevent repeated reports of failure
69 to import a module that can be triggered by ``butler --help``.
70 """
71 try:
72 return doImport(pluginName)
73 except Exception as err:
74 log.warning("Could not import plugin from %s, skipping.", pluginName)
75 log.debug(
76 "Plugin import exception: %s\nTraceback:\n%s",
77 err,
78 "".join(traceback.format_tb(err.__traceback__)),
79 )
80 return None
83class LoaderCLI(click.MultiCommand, abc.ABC):
84 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
85 subcommands at runtime.
86 """
88 def __init__(self, *args: Any, **kwargs: Any) -> None:
89 super().__init__(*args, **kwargs)
91 @property
92 @abc.abstractmethod
93 def localCmdPkg(self) -> str:
94 """Identifies the location of the commands that are in this
95 package.
97 `getLocalCommands` assumes that the commands can be found in
98 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
99 should be overridden.
101 Returns
102 -------
103 package : `str`
104 The fully qualified location of this package.
105 """
106 raise NotImplementedError()
108 def getLocalCommands(self) -> defaultdict[str, list[str]]:
109 """Get the commands offered by the local package. This assumes that the
110 commands can be found in `localCmdPkg.__all__`, if this is not the case
111 then this function should be overridden.
113 Returns
114 -------
115 commands : `defaultdict` [`str`, `list` [`str`]]
116 The key is the command name. The value is a list of package(s) that
117 contains the command.
118 """
119 commandsLocation = _importPlugin(self.localCmdPkg)
120 if commandsLocation is None:
121 # _importPlugins logs an error, don't need to do it again here.
122 return defaultdict(list)
123 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}"
124 return defaultdict(
125 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
126 )
128 def list_commands(self, ctx: click.Context) -> list[str]:
129 """Get all the commands that can be called by the
130 butler command, it is used to generate the --help output.
132 Used by Click.
134 Parameters
135 ----------
136 ctx : `click.Context`
137 The current Click context.
139 Returns
140 -------
141 commands : `list` [`str`]
142 The names of the commands that can be called by the butler command.
143 """
144 self._setupLogging(ctx)
145 commands = self._getCommands()
146 self._raiseIfDuplicateCommands(commands)
147 return sorted(commands)
149 def get_command(self, ctx: click.Context, name: str) -> click.Command | None:
150 """Get a single command for execution.
152 Used by Click.
154 Parameters
155 ----------
156 ctx : `click.Context`
157 The current Click context.
158 name : `str`
159 The name of the command to return.
161 Returns
162 -------
163 command : `click.Command`
164 A Command that wraps a callable command function.
165 """
166 self._setupLogging(ctx)
167 commands = self._getCommands()
168 if name not in commands:
169 return None
170 self._raiseIfDuplicateCommands(commands)
171 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name)
172 # The click.command decorator returns an instance of a class, which
173 # is something that doImport is not expecting. We add it in as an
174 # option here to appease mypy.
175 plugin = _importPlugin(module_str)
176 if not plugin:
177 return None
178 if not isinstance(plugin, click.Command):
179 raise RuntimeError(
180 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}"
181 )
182 return plugin
184 def _setupLogging(self, ctx: click.Context | None) -> None:
185 """Init the logging system and config it for the command.
187 Subcommands may further configure the log settings.
188 """
189 if isinstance(ctx, click.Context):
190 CliLog.initLog(
191 longlog=ctx.params.get(long_log_option.name(), False),
192 log_tty=ctx.params.get(log_tty_option.name(), True),
193 log_file=ctx.params.get(log_file_option.name(), ()),
194 log_label=ctx.params.get(log_label_option.name(), ()),
195 )
196 if log_level_option.name() in ctx.params:
197 CliLog.setLogLevels(ctx.params[log_level_option.name()])
198 else:
199 # This works around a bug in sphinx-click, where it passes in the
200 # click.MultiCommand instead of the context.
201 # https://github.com/click-contrib/sphinx-click/issues/70
202 CliLog.initLog(longlog=False)
203 logging.debug(
204 "The passed-in context was not a click.Context, could not determine --long-log or "
205 "--log-level values."
206 )
208 @classmethod
209 def getPluginList(cls) -> list[str]:
210 """Get the list of importable yaml files that contain cli data for this
211 command.
213 Returns
214 -------
215 `list` [`str`]
216 The list of files that contain yaml data about a cli plugin.
217 """
218 if not hasattr(cls, "pluginEnvVar"):
219 return []
220 pluginModules = os.environ.get(cls.pluginEnvVar)
221 if pluginModules:
222 return [p for p in pluginModules.split(":") if p != ""]
223 return []
225 @classmethod
226 def _funcNameToCmdName(cls, functionName: str) -> str:
227 """Convert function name to the butler command name: change
228 underscores, (used in functions) to dashes (used in commands), and
229 change local-package command names that conflict with python keywords
230 to a legal function name.
231 """
232 return functionName.replace("_", "-")
234 @classmethod
235 def _cmdNameToFuncName(cls, commandName: str) -> str:
236 """Convert butler command name to function name: change dashes (used in
237 commands) to underscores (used in functions), and for local-package
238 commands names that conflict with python keywords, change the local,
239 legal, function name to the command name.
240 """
241 return commandName.replace("-", "_")
243 @staticmethod
244 def _mergeCommandLists(
245 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]]
246 ) -> defaultdict[str, list[str]]:
247 """Combine two dicts whose keys are strings (command name) and values
248 are list of string (the package(s) that provide the named command).
250 Parameters
251 ----------
252 a : `defaultdict` [`str`, `list` [`str`]]
253 The key is the command name. The value is a list of package(s) that
254 contains the command.
255 b : (same as a)
257 Returns
258 -------
259 commands : `defaultdict` [`str`: [`str`]]
260 For convenience, returns a extended with b. ('a' is modified in
261 place.)
262 """
263 for key, val in b.items():
264 a[key].extend(val)
265 return a
267 @classmethod
268 def _getPluginCommands(cls) -> defaultdict[str, list[str]]:
269 """Get the commands offered by plugin packages.
271 Returns
272 -------
273 commands : `defaultdict` [`str`, `list` [`str`]]
274 The key is the command name. The value is a list of package(s) that
275 contains the command.
276 """
277 commands: defaultdict[str, list[str]] = defaultdict(list)
278 for pluginName in cls.getPluginList():
279 try:
280 with open(pluginName) as resourceFile:
281 resources = defaultdict(list, yaml.safe_load(resourceFile))
282 except Exception as err:
283 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
284 continue
285 if "cmd" not in resources:
286 log.warning("No commands found in %s, skipping.", pluginName)
287 continue
288 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
289 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
290 return commands
292 def _getCommands(self) -> defaultdict[str, list[str]]:
293 """Get the commands offered by daf_butler and plugin packages.
295 Returns
296 -------
297 commands : `defaultdict` [`str`, `list` [`str`]]
298 The key is the command name. The value is a list of package(s) that
299 contains the command.
300 """
301 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
303 @staticmethod
304 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None:
305 """If any provided command is offered by more than one package raise an
306 exception.
308 Parameters
309 ----------
310 commands : `defaultdict` [`str`, `list` [`str`]]
311 The key is the command name. The value is a list of package(s) that
312 contains the command.
314 Raises
315 ------
316 click.ClickException
317 Raised if a command is offered by more than one package, with an
318 error message to be displayed to the user.
319 """
320 msg = ""
321 for command, packages in commands.items():
322 if len(packages) > 1:
323 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
324 if msg:
325 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.")
328class ButlerCLI(LoaderCLI):
329 """Specialized command loader implementing the ``butler`` command."""
331 localCmdPkg = "lsst.daf.butler.cli.cmd"
333 pluginEnvVar = "DAF_BUTLER_PLUGINS"
335 @classmethod
336 def _funcNameToCmdName(cls, functionName: str) -> str:
337 # Docstring inherited from base class.
339 # The "import" command name and "butler_import" function name are
340 # defined in cli/cmd/commands.py, and if those names are changed they
341 # must be changed here as well.
342 # It is expected that there will be very few butler command names that
343 # need to be changed because of e.g. conflicts with python keywords (as
344 # is done here and in _cmdNameToFuncName for the 'import' command). If
345 # this becomes a common need then some way of doing this should be
346 # invented that is better than hard coding the function names into
347 # these conversion functions. An extension of the 'cli/resources.yaml'
348 # file (as is currently used in obs_base) might be a good way to do it.
349 if functionName == "butler_import":
350 return "import"
351 return super()._funcNameToCmdName(functionName)
353 @classmethod
354 def _cmdNameToFuncName(cls, commandName: str) -> str:
355 # Docstring inherited from base class.
356 if commandName == "import":
357 return "butler_import"
358 return super()._cmdNameToFuncName(commandName)
361@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
362@log_level_option()
363@long_log_option()
364@log_file_option()
365@log_tty_option()
366@log_label_option()
367@ClickProgressHandler.option
368def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None:
369 """Command line interface for butler.
371 log_level is handled by get_command and list_commands, and is called in
372 one of those functions before this is called. long_log is handled by
373 setup_logging.
374 """
375 pass
378def main() -> click.Command:
379 """Return main entry point for command-line."""
380 return cli()