Coverage for python/lsst/daf/butler/cli/butler.py: 36%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import abc
23import functools
24import logging
25import os
26import traceback
27from collections import defaultdict
29import click
30import yaml
31from lsst.utils import doImport
33from .cliLog import CliLog
34from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option
35from .progress import ClickProgressHandler
37log = logging.getLogger(__name__)
40@functools.lru_cache
41def _importPlugin(pluginName):
42 """Import a plugin that contains Click commands.
44 Parameters
45 ----------
46 pluginName : `str`
47 An importable module whose __all__ parameter contains the commands
48 that can be called.
50 Returns
51 -------
52 An imported module or None
53 The imported module, or None if the module could not be imported.
55 Notes
56 -----
57 A cache is used in order to prevent repeated reports of failure
58 to import a module that can be triggered by ``butler --help``.
59 """
60 try:
61 return doImport(pluginName)
62 except Exception as err:
63 log.warning("Could not import plugin from %s, skipping.", pluginName)
64 log.debug(
65 "Plugin import exception: %s\nTraceback:\n%s",
66 err,
67 "".join(traceback.format_tb(err.__traceback__)),
68 )
69 return None
72class LoaderCLI(click.MultiCommand, abc.ABC):
73 """Extends `click.MultiCommand`, which dispatches to subcommands, to load
74 subcommands at runtime."""
76 def __init__(self, *args, **kwargs):
77 super().__init__(*args, **kwargs)
79 @property
80 @abc.abstractmethod
81 def localCmdPkg(self):
82 """localCmdPkg identifies the location of the commands that are in this
83 package. `getLocalCommands` assumes that the commands can be found in
84 `localCmdPkg.__all__`, if this is not the case then getLocalCommands
85 should be overridden.
87 Returns
88 -------
89 package : `str`
90 The fully qualified location of this package.
91 """
92 pass
94 def getLocalCommands(self):
95 """Get the commands offered by the local package. This assumes that the
96 commands can be found in `localCmdPkg.__all__`, if this is not the case
97 then this function should be overridden.
99 Returns
100 -------
101 commands : `defaultdict` [`str`, `list` [`str`]]
102 The key is the command name. The value is a list of package(s) that
103 contains the command.
104 """
105 commandsLocation = _importPlugin(self.localCmdPkg)
106 if commandsLocation is None:
107 # _importPlugins logs an error, don't need to do it again here.
108 return defaultdict(list)
109 return defaultdict(
110 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__}
111 )
113 def list_commands(self, ctx):
114 """Used by Click to get all the commands that can be called by the
115 butler command, it is used to generate the --help output.
117 Parameters
118 ----------
119 ctx : `click.Context`
120 The current Click context.
122 Returns
123 -------
124 commands : `list` [`str`]
125 The names of the commands that can be called by the butler command.
126 """
127 self._setupLogging(ctx)
128 commands = self._getCommands()
129 self._raiseIfDuplicateCommands(commands)
130 return sorted(commands)
132 def get_command(self, ctx, name):
133 """Used by Click to get a single command for execution.
135 Parameters
136 ----------
137 ctx : `click.Context`
138 The current Click context.
139 name : `str`
140 The name of the command to return.
142 Returns
143 -------
144 command : `click.Command`
145 A Command that wraps a callable command function.
146 """
147 self._setupLogging(ctx)
148 commands = self._getCommands()
149 if name not in commands:
150 return None
151 self._raiseIfDuplicateCommands(commands)
152 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name))
154 def _setupLogging(self, ctx):
155 """Init the logging system and config it for the command.
157 Subcommands may further configure the log settings."""
158 if isinstance(ctx, click.Context):
159 CliLog.initLog(
160 longlog=ctx.params.get(long_log_option.name(), False),
161 log_tty=ctx.params.get(log_tty_option.name(), True),
162 log_file=ctx.params.get(log_file_option.name(), ()),
163 log_label=ctx.params.get(log_label_option.name(), ()),
164 )
165 if log_level_option.name() in ctx.params:
166 CliLog.setLogLevels(ctx.params[log_level_option.name()])
167 else:
168 # This works around a bug in sphinx-click, where it passes in the
169 # click.MultiCommand instead of the context.
170 # https://github.com/click-contrib/sphinx-click/issues/70
171 CliLog.initLog(longlog=False)
172 logging.debug(
173 "The passed-in context was not a click.Context, could not determine --long-log or "
174 "--log-level values."
175 )
177 @classmethod
178 def getPluginList(cls):
179 """Get the list of importable yaml files that contain cli data for this
180 command.
182 Returns
183 -------
184 `list` [`str`]
185 The list of files that contain yaml data about a cli plugin.
186 """
187 if not hasattr(cls, "pluginEnvVar"):
188 return []
189 pluginModules = os.environ.get(cls.pluginEnvVar)
190 if pluginModules:
191 return [p for p in pluginModules.split(":") if p != ""]
192 return []
194 @classmethod
195 def _funcNameToCmdName(cls, functionName):
196 """Convert function name to the butler command name: change
197 underscores, (used in functions) to dashes (used in commands), and
198 change local-package command names that conflict with python keywords
199 to a legal function name.
200 """
201 return functionName.replace("_", "-")
203 @classmethod
204 def _cmdNameToFuncName(cls, commandName):
205 """Convert butler command name to function name: change dashes (used in
206 commands) to underscores (used in functions), and for local-package
207 commands names that conflict with python keywords, change the local,
208 legal, function name to the command name."""
209 return commandName.replace("-", "_")
211 @staticmethod
212 def _mergeCommandLists(a, b):
213 """Combine two dicts whose keys are strings (command name) and values
214 are list of string (the package(s) that provide the named command).
216 Parameters
217 ----------
218 a : `defaultdict` [`str`, `list` [`str`]]
219 The key is the command name. The value is a list of package(s) that
220 contains the command.
221 b : (same as a)
223 Returns
224 -------
225 commands : `defaultdict` [`str`: [`str`]]
226 For convenience, returns a extended with b. ('a' is modified in
227 place.)
228 """
229 for key, val in b.items():
230 a[key].extend(val)
231 return a
233 @classmethod
234 def _getPluginCommands(cls):
235 """Get the commands offered by plugin packages.
237 Returns
238 -------
239 commands : `defaultdict` [`str`, `list` [`str`]]
240 The key is the command name. The value is a list of package(s) that
241 contains the command.
242 """
243 commands = defaultdict(list)
244 for pluginName in cls.getPluginList():
245 try:
246 with open(pluginName, "r") as resourceFile:
247 resources = defaultdict(list, yaml.safe_load(resourceFile))
248 except Exception as err:
249 log.warning("Error loading commands from %s, skipping. %s", pluginName, err)
250 continue
251 if "cmd" not in resources:
252 log.warning("No commands found in %s, skipping.", pluginName)
253 continue
254 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]}
255 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands))
256 return commands
258 def _getCommands(self):
259 """Get the commands offered by daf_butler and plugin packages.
261 Returns
262 -------
263 commands : `defaultdict` [`str`, `list` [`str`]]
264 The key is the command name. The value is a list of package(s) that
265 contains the command.
266 """
267 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands())
269 @staticmethod
270 def _raiseIfDuplicateCommands(commands):
271 """If any provided command is offered by more than one package raise an
272 exception.
274 Parameters
275 ----------
276 commands : `defaultdict` [`str`, `list` [`str`]]
277 The key is the command name. The value is a list of package(s) that
278 contains the command.
280 Raises
281 ------
282 click.ClickException
283 Raised if a command is offered by more than one package, with an
284 error message to be displayed to the user.
285 """
287 msg = ""
288 for command, packages in commands.items():
289 if len(packages) > 1:
290 msg += f"Command '{command}' exists in packages {', '.join(packages)}. "
291 if msg:
292 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.")
295class ButlerCLI(LoaderCLI):
297 localCmdPkg = "lsst.daf.butler.cli.cmd"
299 pluginEnvVar = "DAF_BUTLER_PLUGINS"
301 @classmethod
302 def _funcNameToCmdName(cls, functionName):
303 # Docstring inherited from base class.
305 # The "import" command name and "butler_import" function name are
306 # defined in cli/cmd/commands.py, and if those names are changed they
307 # must be changed here as well.
308 # It is expected that there will be very few butler command names that
309 # need to be changed because of e.g. conflicts with python keywords (as
310 # is done here and in _cmdNameToFuncName for the 'import' command). If
311 # this becomes a common need then some way of doing this should be
312 # invented that is better than hard coding the function names into
313 # these conversion functions. An extension of the 'cli/resources.yaml'
314 # file (as is currently used in obs_base) might be a good way to do it.
315 if functionName == "butler_import":
316 return "import"
317 return super()._funcNameToCmdName(functionName)
319 @classmethod
320 def _cmdNameToFuncName(cls, commandName):
321 # Docstring inherited from base class.
322 if commandName == "import":
323 return "butler_import"
324 return super()._cmdNameToFuncName(commandName)
327@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"]))
328@log_level_option()
329@long_log_option()
330@log_file_option()
331@log_tty_option()
332@log_label_option()
333@ClickProgressHandler.option
334def cli(log_level, long_log, log_file, log_tty, log_label, progress):
335 # log_level is handled by get_command and list_commands, and is called in
336 # one of those functions before this is called. long_log is handled by
337 # setup_logging.
338 pass
341def main():
342 return cli()