Coverage for python/lsst/daf/butler/cli/butler.py: 46%
131 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-27 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "LoaderCLI",
31 "ButlerCLI",
32 "cli",
33 "main",
34)
37import abc
38import functools
39import logging
40import os
41import traceback
42import types
43from collections import defaultdict
44from typing import Any
46import click
47import yaml
48from lsst.utils import doImport
50from .cliLog import CliLog
51from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
52from .progress import ClickProgressHandler
54log = logging.getLogger(__name__)
57@functools.lru_cache
58def _importPlugin(pluginName: str) -> types.ModuleType | type | None | click.Command:
59 """Import a plugin that contains Click commands.
61 Parameters
62 ----------
63 pluginName : `str`
64 An importable module whose __all__ parameter contains the commands
65 that can be called.
67 Returns
68 -------
69 An imported module or None
70 The imported module, or None if the module could not be imported.
72 Notes
73 -----
74 A cache is used in order to prevent repeated reports of failure
75 to import a module that can be triggered by ``butler --help``.
76 """
77 try:
78 return doImport(pluginName)
79 except Exception as err:
80 log.warning("Could not import plugin from %s, skipping.", pluginName)
81 log.debug(
82 "Plugin import exception: %s\nTraceback:\n%s",
83 err,
84 "".join(traceback.format_tb(err.__traceback__)),
85 )
86 return None
89class LoaderCLI(click.MultiCommand, abc.ABC):
90 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
91 subcommands at runtime.
92 """
94 def __init__(self, *args: Any, **kwargs: Any) -> None:
95 super().__init__(*args, **kwargs)
97 @property
98 @abc.abstractmethod
99 def localCmdPkg(self) -> str:
100 """Identifies the location of the commands that are in this
101 package.
103 `getLocalCommands` assumes that the commands can be found in
104 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
105 should be overridden.
107 Returns
108 -------
109 package : `str`
110 The fully qualified location of this package.
111 """
112 raise NotImplementedError()
114 def getLocalCommands(self) -> defaultdict[str, list[str]]:
115 """Get the commands offered by the local package. This assumes that the
116 commands can be found in `localCmdPkg.__all__`, if this is not the case
117 then this function should be overridden.
119 Returns
120 -------
121 commands : `defaultdict` [`str`, `list` [`str`]]
122 The key is the command name. The value is a list of package(s) that
123 contains the command.
124 """
125 commandsLocation = _importPlugin(self.localCmdPkg)
126 if commandsLocation is None:
127 # _importPlugins logs an error, don't need to do it again here.
128 return defaultdict(list)
129 assert hasattr(commandsLocation, "__all__"), f"Must define __all__ in {commandsLocation}"
130 return defaultdict(
131 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
132 )
134 def list_commands(self, ctx: click.Context) -> list[str]:
135 """Get all the commands that can be called by the
136 butler command, it is used to generate the --help output.
138 Used by Click.
140 Parameters
141 ----------
142 ctx : `click.Context`
143 The current Click context.
145 Returns
146 -------
147 commands : `list` [`str`]
148 The names of the commands that can be called by the butler command.
149 """
150 self._setupLogging(ctx)
151 commands = self._getCommands()
152 self._raiseIfDuplicateCommands(commands)
153 return sorted(commands)
155 def get_command(self, ctx: click.Context, name: str) -> click.Command | None:
156 """Get a single command for execution.
158 Used by Click.
160 Parameters
161 ----------
162 ctx : `click.Context`
163 The current Click context.
164 name : `str`
165 The name of the command to return.
167 Returns
168 -------
169 command : `click.Command`
170 A Command that wraps a callable command function.
171 """
172 self._setupLogging(ctx)
173 commands = self._getCommands()
174 if name not in commands:
175 return None
176 self._raiseIfDuplicateCommands(commands)
177 module_str = commands[name][0] + "." + self._cmdNameToFuncName(name)
178 # The click.command decorator returns an instance of a class, which
179 # is something that doImport is not expecting. We add it in as an
180 # option here to appease mypy.
181 plugin = _importPlugin(module_str)
182 if not plugin:
183 return None
184 if not isinstance(plugin, click.Command):
185 raise RuntimeError(
186 f"Command {name!r} loaded from {module_str} is not a click Command, is {type(plugin)}"
187 )
188 return plugin
190 def _setupLogging(self, ctx: click.Context | None) -> None:
191 """Init the logging system and config it for the command.
193 Subcommands may further configure the log settings.
194 """
195 if isinstance(ctx, click.Context):
196 CliLog.initLog(
197 longlog=ctx.params.get(long_log_option.name(), False),
198 log_tty=ctx.params.get(log_tty_option.name(), True),
199 log_file=ctx.params.get(log_file_option.name(), ()),
200 log_label=ctx.params.get(log_label_option.name(), ()),
201 )
202 if log_level_option.name() in ctx.params:
203 CliLog.setLogLevels(ctx.params[log_level_option.name()])
204 else:
205 # This works around a bug in sphinx-click, where it passes in the
206 # click.MultiCommand instead of the context.
207 # https://github.com/click-contrib/sphinx-click/issues/70
208 CliLog.initLog(longlog=False)
209 logging.debug(
210 "The passed-in context was not a click.Context, could not determine --long-log or "
211 "--log-level values."
212 )
214 @classmethod
215 def getPluginList(cls) -> list[str]:
216 """Get the list of importable yaml files that contain cli data for this
217 command.
219 Returns
220 -------
221 `list` [`str`]
222 The list of files that contain yaml data about a cli plugin.
223 """
224 if not hasattr(cls, "pluginEnvVar"):
225 return []
226 pluginModules = os.environ.get(cls.pluginEnvVar)
227 if pluginModules:
228 return [p for p in pluginModules.split(":") if p != ""]
229 return []
231 @classmethod
232 def _funcNameToCmdName(cls, functionName: str) -> str:
233 """Convert function name to the butler command name: change
234 underscores, (used in functions) to dashes (used in commands), and
235 change local-package command names that conflict with python keywords
236 to a legal function name.
237 """
238 return functionName.replace("_", "-")
240 @classmethod
241 def _cmdNameToFuncName(cls, commandName: str) -> str:
242 """Convert butler command name to function name: change dashes (used in
243 commands) to underscores (used in functions), and for local-package
244 commands names that conflict with python keywords, change the local,
245 legal, function name to the command name.
246 """
247 return commandName.replace("-", "_")
249 @staticmethod
250 def _mergeCommandLists(
251 a: defaultdict[str, list[str]], b: defaultdict[str, list[str]]
252 ) -> defaultdict[str, list[str]]:
253 """Combine two dicts whose keys are strings (command name) and values
254 are list of string (the package(s) that provide the named command).
256 Parameters
257 ----------
258 a : `defaultdict` [`str`, `list` [`str`]]
259 The key is the command name. The value is a list of package(s) that
260 contains the command.
261 b : (same as a)
263 Returns
264 -------
265 commands : `defaultdict` [`str`: [`str`]]
266 For convenience, returns a extended with b. ('a' is modified in
267 place.)
268 """
269 for key, val in b.items():
270 a[key].extend(val)
271 return a
273 @classmethod
274 def _getPluginCommands(cls) -> defaultdict[str, list[str]]:
275 """Get the commands offered by plugin packages.
277 Returns
278 -------
279 commands : `defaultdict` [`str`, `list` [`str`]]
280 The key is the command name. The value is a list of package(s) that
281 contains the command.
282 """
283 commands: defaultdict[str, list[str]] = defaultdict(list)
284 for pluginName in cls.getPluginList():
285 try:
286 with open(pluginName) as resourceFile:
287 resources = defaultdict(list, yaml.safe_load(resourceFile))
288 except Exception as err:
289 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
290 continue
291 if "cmd" not in resources:
292 log.warning("No commands found in %s, skipping.", pluginName)
293 continue
294 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
295 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
296 return commands
298 def _getCommands(self) -> defaultdict[str, list[str]]:
299 """Get the commands offered by daf_butler and plugin packages.
301 Returns
302 -------
303 commands : `defaultdict` [`str`, `list` [`str`]]
304 The key is the command name. The value is a list of package(s) that
305 contains the command.
306 """
307 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
309 @staticmethod
310 def _raiseIfDuplicateCommands(commands: defaultdict[str, list[str]]) -> None:
311 """If any provided command is offered by more than one package raise an
312 exception.
314 Parameters
315 ----------
316 commands : `defaultdict` [`str`, `list` [`str`]]
317 The key is the command name. The value is a list of package(s) that
318 contains the command.
320 Raises
321 ------
322 click.ClickException
323 Raised if a command is offered by more than one package, with an
324 error message to be displayed to the user.
325 """
326 msg = ""
327 for command, packages in commands.items():
328 if len(packages) > 1:
329 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
330 if msg:
331 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.")
334class ButlerCLI(LoaderCLI):
335 """Specialized command loader implementing the ``butler`` command."""
337 localCmdPkg = "lsst.daf.butler.cli.cmd"
339 pluginEnvVar = "DAF_BUTLER_PLUGINS"
341 @classmethod
342 def _funcNameToCmdName(cls, functionName: str) -> str:
343 # Docstring inherited from base class.
345 # The "import" command name and "butler_import" function name are
346 # defined in cli/cmd/commands.py, and if those names are changed they
347 # must be changed here as well.
348 # It is expected that there will be very few butler command names that
349 # need to be changed because of e.g. conflicts with python keywords (as
350 # is done here and in _cmdNameToFuncName for the 'import' command). If
351 # this becomes a common need then some way of doing this should be
352 # invented that is better than hard coding the function names into
353 # these conversion functions. An extension of the 'cli/resources.yaml'
354 # file (as is currently used in obs_base) might be a good way to do it.
355 if functionName == "butler_import":
356 return "import"
357 return super()._funcNameToCmdName(functionName)
359 @classmethod
360 def _cmdNameToFuncName(cls, commandName: str) -> str:
361 # Docstring inherited from base class.
362 if commandName == "import":
363 return "butler_import"
364 return super()._cmdNameToFuncName(commandName)
367@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
368@log_level_option()
369@long_log_option()
370@log_file_option()
371@log_tty_option()
372@log_label_option()
373@ClickProgressHandler.option
374def cli(log_level: str, long_log: bool, log_file: str, log_tty: bool, log_label: str, progress: bool) -> None:
375 """Command line interface for butler.
377 log_level is handled by get_command and list_commands, and is called in
378 one of those functions before this is called. long_log is handled by
379 setup_logging.
380 """
381 pass
384def main() -> click.Command:
385 """Return main entry point for command-line."""
386 return cli()