Coverage for python/lsst/daf/butler/cli/butler.py : 34%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import abc
23import click
24from collections import defaultdict
25import functools
26import logging
27import os
28import traceback
29import yaml
31from .cliLog import CliLog
32from .opt import log_level_option, long_log_option
33from .progress import ClickProgressHandler
34from lsst.utils import doImport
37log = logging.getLogger(__name__)
40@functools.lru_cache
41def _importPlugin(pluginName):
42 """Import a plugin that contains Click commands.
44 Parameters
45 ----------
46 pluginName : `str`
47 An importable module whose __all__ parameter contains the commands
48 that can be called.
50 Returns
51 -------
52 An imported module or None
53 The imported module, or None if the module could not be imported.
55 Notes
56 -----
57 A cache is used in order to prevent repeated reports of failure
58 to import a module that can be triggered by ``butler --help``.
59 """
60 try:
61 return doImport(pluginName)
62 except Exception as err:
63 log.warning("Could not import plugin from %s, skipping.", pluginName)
64 log.debug("Plugin import exception: %s\nTraceback:\n%s", err,
65 "".join(traceback.format_tb(err.__traceback__)))
66 return None
69class LoaderCLI(click.MultiCommand, abc.ABC):
70 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
71 subcommands at runtime."""
73 def __init__(self, *args, **kwargs):
74 super().__init__(*args, **kwargs)
76 @property
77 @abc.abstractmethod
78 def localCmdPkg(self):
79 """localCmdPkg identifies the location of the commands that are in this
80 package. `getLocalCommands` assumes that the commands can be found in
81 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
82 should be overridden.
84 Returns
85 -------
86 package : `str`
87 The fully qualified location of this package.
88 """
89 pass
91 def getLocalCommands(self):
92 """Get the commands offered by the local package. This assumes that the
93 commands can be found in `localCmdPkg.__all__`, if this is not the case
94 then this function should be overridden.
96 Returns
97 -------
98 commands : `defaultdict` [`str`, `list` [`str`]]
99 The key is the command name. The value is a list of package(s) that
100 contains the command.
101 """
102 commandsLocation = _importPlugin(self.localCmdPkg)
103 if commandsLocation is None:
104 # _importPlugins logs an error, don't need to do it again here.
105 return defaultdict(list)
106 return defaultdict(list, {self._funcNameToCmdName(f):
107 [self.localCmdPkg] for f in commandsLocation.__all__})
109 def list_commands(self, ctx):
110 """Used by Click to get all the commands that can be called by the
111 butler command, it is used to generate the --help output.
113 Parameters
114 ----------
115 ctx : `click.Context`
116 The current Click context.
118 Returns
119 -------
120 commands : `list` [`str`]
121 The names of the commands that can be called by the butler command.
122 """
123 self._setupLogging(ctx)
124 commands = self._getCommands()
125 self._raiseIfDuplicateCommands(commands)
126 return sorted(commands)
128 def get_command(self, ctx, name):
129 """Used by Click to get a single command for execution.
131 Parameters
132 ----------
133 ctx : `click.Context`
134 The current Click context.
135 name : `str`
136 The name of the command to return.
138 Returns
139 -------
140 command : `click.Command`
141 A Command that wraps a callable command function.
142 """
143 self._setupLogging(ctx)
144 commands = self._getCommands()
145 if name not in commands:
146 return None
147 self._raiseIfDuplicateCommands(commands)
148 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name))
150 def _setupLogging(self, ctx):
151 """Init the logging system and config it for the command.
153 Subcommands may further configure the log settings."""
154 if isinstance(ctx, click.Context):
155 CliLog.initLog(longlog=ctx.params.get(long_log_option.name(), False))
156 if log_level_option.name() in ctx.params:
157 CliLog.setLogLevels(ctx.params[log_level_option.name()])
158 else:
159 # This works around a bug in sphinx-click, where it passes in the
160 # click.MultiCommand instead of the context.
161 # https://github.com/click-contrib/sphinx-click/issues/70
162 CliLog.initLog(longlog=False)
163 logging.debug("The passed-in context was not a click.Context, could not determine --long-log or "
164 "--log-level values.")
166 @classmethod
167 def getPluginList(cls):
168 """Get the list of importable yaml files that contain cli data for this
169 command.
171 Returns
172 -------
173 `list` [`str`]
174 The list of files that contain yaml data about a cli plugin.
175 """
176 if not hasattr(cls, "pluginEnvVar"):
177 return []
178 pluginModules = os.environ.get(cls.pluginEnvVar)
179 if pluginModules:
180 return [p for p in pluginModules.split(":") if p != '']
181 return []
183 @classmethod
184 def _funcNameToCmdName(cls, functionName):
185 """Convert function name to the butler command name: change
186 underscores, (used in functions) to dashes (used in commands), and
187 change local-package command names that conflict with python keywords
188 to a legal function name.
189 """
190 return functionName.replace("_", "-")
192 @classmethod
193 def _cmdNameToFuncName(cls, commandName):
194 """Convert butler command name to function name: change dashes (used in
195 commands) to underscores (used in functions), and for local-package
196 commands names that conflict with python keywords, change the local,
197 legal, function name to the command name."""
198 return commandName.replace("-", "_")
200 @staticmethod
201 def _mergeCommandLists(a, b):
202 """Combine two dicts whose keys are strings (command name) and values
203 are list of string (the package(s) that provide the named command).
205 Parameters
206 ----------
207 a : `defaultdict` [`str`, `list` [`str`]]
208 The key is the command name. The value is a list of package(s) that
209 contains the command.
210 b : (same as a)
212 Returns
213 -------
214 commands : `defaultdict` [`str`: [`str`]]
215 For convenience, returns a extended with b. ('a' is modified in
216 place.)
217 """
218 for key, val in b.items():
219 a[key].extend(val)
220 return a
222 @classmethod
223 def _getPluginCommands(cls):
224 """Get the commands offered by plugin packages.
226 Returns
227 -------
228 commands : `defaultdict` [`str`, `list` [`str`]]
229 The key is the command name. The value is a list of package(s) that
230 contains the command.
231 """
232 commands = defaultdict(list)
233 for pluginName in cls.getPluginList():
234 try:
235 with open(pluginName, "r") as resourceFile:
236 resources = defaultdict(list, yaml.safe_load(resourceFile))
237 except Exception as err:
238 log.warning(f"Error loading commands from {pluginName}, skipping. {err}")
239 continue
240 if "cmd" not in resources:
241 log.warning(f"No commands found in {pluginName}, skipping.")
242 continue
243 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
244 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
245 return commands
247 def _getCommands(self):
248 """Get the commands offered by daf_butler and plugin packages.
250 Returns
251 -------
252 commands : `defaultdict` [`str`, `list` [`str`]]
253 The key is the command name. The value is a list of package(s) that
254 contains the command.
255 """
256 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
258 @staticmethod
259 def _raiseIfDuplicateCommands(commands):
260 """If any provided command is offered by more than one package raise an
261 exception.
263 Parameters
264 ----------
265 commands : `defaultdict` [`str`, `list` [`str`]]
266 The key is the command name. The value is a list of package(s) that
267 contains the command.
269 Raises
270 ------
271 click.ClickException
272 Raised if a command is offered by more than one package, with an
273 error message to be displayed to the user.
274 """
276 msg = ""
277 for command, packages in commands.items():
278 if len(packages) > 1:
279 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
280 if msg:
281 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.")
284class ButlerCLI(LoaderCLI):
286 localCmdPkg = "lsst.daf.butler.cli.cmd"
288 pluginEnvVar = "DAF_BUTLER_PLUGINS"
290 @classmethod
291 def _funcNameToCmdName(cls, functionName):
292 # Docstring inherited from base class.
294 # The "import" command name and "butler_import" function name are
295 # defined in cli/cmd/commands.py, and if those names are changed they
296 # must be changed here as well.
297 # It is expected that there will be very few butler command names that
298 # need to be changed because of e.g. conflicts with python keywords (as
299 # is done here and in _cmdNameToFuncName for the 'import' command). If
300 # this becomes a common need then some way of doing this should be
301 # invented that is better than hard coding the function names into
302 # these conversion functions. An extension of the 'cli/resources.yaml'
303 # file (as is currently used in obs_base) might be a good way to do it.
304 if functionName == "butler_import":
305 return "import"
306 return super()._funcNameToCmdName(functionName)
308 @classmethod
309 def _cmdNameToFuncName(cls, commandName):
310 # Docstring inherited from base class.
311 if commandName == "import":
312 return "butler_import"
313 return super()._cmdNameToFuncName(commandName)
316@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
317@log_level_option()
318@long_log_option()
319@ClickProgressHandler.option
320def cli(log_level, long_log, progress):
321 # log_level is handled by get_command and list_commands, and is called in
322 # one of those functions before this is called. long_log is handled by
323 # setup_logging.
324 pass
327def main():
328 return cli()