Coverage for python/lsst/daf/butler/cli/butler.py: 36%
122 statements
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 12:04 -0700
« prev ^ index » next coverage.py v6.4.4, created at 2022-08-19 12:04 -0700
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
23__all__ = (
24 "LoaderCLI",
25 "ButlerCLI",
26 "cli",
27 "main",
28)
31import abc
32import functools
33import logging
34import os
35import traceback
36from collections import defaultdict
38import click
39import yaml
40from lsst.utils import doImport
42from .cliLog import CliLog
43from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
44from .progress import ClickProgressHandler
46log = logging.getLogger(__name__)
49@functools.lru_cache
50def _importPlugin(pluginName):
51 """Import a plugin that contains Click commands.
53 Parameters
54 ----------
55 pluginName : `str`
56 An importable module whose __all__ parameter contains the commands
57 that can be called.
59 Returns
60 -------
61 An imported module or None
62 The imported module, or None if the module could not be imported.
64 Notes
65 -----
66 A cache is used in order to prevent repeated reports of failure
67 to import a module that can be triggered by ``butler --help``.
68 """
69 try:
70 return doImport(pluginName)
71 except Exception as err:
72 log.warning("Could not import plugin from %s, skipping.", pluginName)
73 log.debug(
74 "Plugin import exception: %s\nTraceback:\n%s",
75 err,
76 "".join(traceback.format_tb(err.__traceback__)),
77 )
78 return None
81class LoaderCLI(click.MultiCommand, abc.ABC):
82 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
83 subcommands at runtime."""
85 def __init__(self, *args, **kwargs):
86 super().__init__(*args, **kwargs)
88 @property
89 @abc.abstractmethod
90 def localCmdPkg(self):
91 """localCmdPkg identifies the location of the commands that are in this
92 package. `getLocalCommands` assumes that the commands can be found in
93 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
94 should be overridden.
96 Returns
97 -------
98 package : `str`
99 The fully qualified location of this package.
100 """
101 pass
103 def getLocalCommands(self):
104 """Get the commands offered by the local package. This assumes that the
105 commands can be found in `localCmdPkg.__all__`, if this is not the case
106 then this function should be overridden.
108 Returns
109 -------
110 commands : `defaultdict` [`str`, `list` [`str`]]
111 The key is the command name. The value is a list of package(s) that
112 contains the command.
113 """
114 commandsLocation = _importPlugin(self.localCmdPkg)
115 if commandsLocation is None:
116 # _importPlugins logs an error, don't need to do it again here.
117 return defaultdict(list)
118 return defaultdict(
119 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
120 )
122 def list_commands(self, ctx):
123 """Used by Click to get all the commands that can be called by the
124 butler command, it is used to generate the --help output.
126 Parameters
127 ----------
128 ctx : `click.Context`
129 The current Click context.
131 Returns
132 -------
133 commands : `list` [`str`]
134 The names of the commands that can be called by the butler command.
135 """
136 self._setupLogging(ctx)
137 commands = self._getCommands()
138 self._raiseIfDuplicateCommands(commands)
139 return sorted(commands)
141 def get_command(self, ctx, name):
142 """Used by Click to get a single command for execution.
144 Parameters
145 ----------
146 ctx : `click.Context`
147 The current Click context.
148 name : `str`
149 The name of the command to return.
151 Returns
152 -------
153 command : `click.Command`
154 A Command that wraps a callable command function.
155 """
156 self._setupLogging(ctx)
157 commands = self._getCommands()
158 if name not in commands:
159 return None
160 self._raiseIfDuplicateCommands(commands)
161 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name))
163 def _setupLogging(self, ctx):
164 """Init the logging system and config it for the command.
166 Subcommands may further configure the log settings."""
167 if isinstance(ctx, click.Context):
168 CliLog.initLog(
169 longlog=ctx.params.get(long_log_option.name(), False),
170 log_tty=ctx.params.get(log_tty_option.name(), True),
171 log_file=ctx.params.get(log_file_option.name(), ()),
172 log_label=ctx.params.get(log_label_option.name(), ()),
173 )
174 if log_level_option.name() in ctx.params:
175 CliLog.setLogLevels(ctx.params[log_level_option.name()])
176 else:
177 # This works around a bug in sphinx-click, where it passes in the
178 # click.MultiCommand instead of the context.
179 # https://github.com/click-contrib/sphinx-click/issues/70
180 CliLog.initLog(longlog=False)
181 logging.debug(
182 "The passed-in context was not a click.Context, could not determine --long-log or "
183 "--log-level values."
184 )
186 @classmethod
187 def getPluginList(cls):
188 """Get the list of importable yaml files that contain cli data for this
189 command.
191 Returns
192 -------
193 `list` [`str`]
194 The list of files that contain yaml data about a cli plugin.
195 """
196 if not hasattr(cls, "pluginEnvVar"):
197 return []
198 pluginModules = os.environ.get(cls.pluginEnvVar)
199 if pluginModules:
200 return [p for p in pluginModules.split(":") if p != ""]
201 return []
203 @classmethod
204 def _funcNameToCmdName(cls, functionName):
205 """Convert function name to the butler command name: change
206 underscores, (used in functions) to dashes (used in commands), and
207 change local-package command names that conflict with python keywords
208 to a legal function name.
209 """
210 return functionName.replace("_", "-")
212 @classmethod
213 def _cmdNameToFuncName(cls, commandName):
214 """Convert butler command name to function name: change dashes (used in
215 commands) to underscores (used in functions), and for local-package
216 commands names that conflict with python keywords, change the local,
217 legal, function name to the command name."""
218 return commandName.replace("-", "_")
220 @staticmethod
221 def _mergeCommandLists(a, b):
222 """Combine two dicts whose keys are strings (command name) and values
223 are list of string (the package(s) that provide the named command).
225 Parameters
226 ----------
227 a : `defaultdict` [`str`, `list` [`str`]]
228 The key is the command name. The value is a list of package(s) that
229 contains the command.
230 b : (same as a)
232 Returns
233 -------
234 commands : `defaultdict` [`str`: [`str`]]
235 For convenience, returns a extended with b. ('a' is modified in
236 place.)
237 """
238 for key, val in b.items():
239 a[key].extend(val)
240 return a
242 @classmethod
243 def _getPluginCommands(cls):
244 """Get the commands offered by plugin packages.
246 Returns
247 -------
248 commands : `defaultdict` [`str`, `list` [`str`]]
249 The key is the command name. The value is a list of package(s) that
250 contains the command.
251 """
252 commands = defaultdict(list)
253 for pluginName in cls.getPluginList():
254 try:
255 with open(pluginName, "r") as resourceFile:
256 resources = defaultdict(list, yaml.safe_load(resourceFile))
257 except Exception as err:
258 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
259 continue
260 if "cmd" not in resources:
261 log.warning("No commands found in %s, skipping.", pluginName)
262 continue
263 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
264 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
265 return commands
267 def _getCommands(self):
268 """Get the commands offered by daf_butler and plugin packages.
270 Returns
271 -------
272 commands : `defaultdict` [`str`, `list` [`str`]]
273 The key is the command name. The value is a list of package(s) that
274 contains the command.
275 """
276 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
278 @staticmethod
279 def _raiseIfDuplicateCommands(commands):
280 """If any provided command is offered by more than one package raise an
281 exception.
283 Parameters
284 ----------
285 commands : `defaultdict` [`str`, `list` [`str`]]
286 The key is the command name. The value is a list of package(s) that
287 contains the command.
289 Raises
290 ------
291 click.ClickException
292 Raised if a command is offered by more than one package, with an
293 error message to be displayed to the user.
294 """
296 msg = ""
297 for command, packages in commands.items():
298 if len(packages) > 1:
299 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
300 if msg:
301 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.")
304class ButlerCLI(LoaderCLI):
306 localCmdPkg = "lsst.daf.butler.cli.cmd"
308 pluginEnvVar = "DAF_BUTLER_PLUGINS"
310 @classmethod
311 def _funcNameToCmdName(cls, functionName):
312 # Docstring inherited from base class.
314 # The "import" command name and "butler_import" function name are
315 # defined in cli/cmd/commands.py, and if those names are changed they
316 # must be changed here as well.
317 # It is expected that there will be very few butler command names that
318 # need to be changed because of e.g. conflicts with python keywords (as
319 # is done here and in _cmdNameToFuncName for the 'import' command). If
320 # this becomes a common need then some way of doing this should be
321 # invented that is better than hard coding the function names into
322 # these conversion functions. An extension of the 'cli/resources.yaml'
323 # file (as is currently used in obs_base) might be a good way to do it.
324 if functionName == "butler_import":
325 return "import"
326 return super()._funcNameToCmdName(functionName)
328 @classmethod
329 def _cmdNameToFuncName(cls, commandName):
330 # Docstring inherited from base class.
331 if commandName == "import":
332 return "butler_import"
333 return super()._cmdNameToFuncName(commandName)
336@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
337@log_level_option()
338@long_log_option()
339@log_file_option()
340@log_tty_option()
341@log_label_option()
342@ClickProgressHandler.option
343def cli(log_level, long_log, log_file, log_tty, log_label, progress):
344 # log_level is handled by get_command and list_commands, and is called in
345 # one of those functions before this is called. long_log is handled by
346 # setup_logging.
347 pass
350def main():
351 return cli()