Coverage for python/lsst/daf/butler/cli/butler.py: 35%

121 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-12-01 19:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import abc 

23import click 

24from collections import defaultdict 

25import functools 

26import logging 

27import os 

28import traceback 

29import yaml 

30 

31from .cliLog import CliLog 

32from .opt import log_level_option, long_log_option, log_file_option, log_tty_option, log_label_option 

33from .progress import ClickProgressHandler 

34from lsst.utils import doImport 

35 

36 

37log = logging.getLogger(__name__) 

38 

39 

40@functools.lru_cache 

41def _importPlugin(pluginName): 

42 """Import a plugin that contains Click commands. 

43 

44 Parameters 

45 ---------- 

46 pluginName : `str` 

47 An importable module whose __all__ parameter contains the commands 

48 that can be called. 

49 

50 Returns 

51 ------- 

52 An imported module or None 

53 The imported module, or None if the module could not be imported. 

54 

55 Notes 

56 ----- 

57 A cache is used in order to prevent repeated reports of failure 

58 to import a module that can be triggered by ``butler --help``. 

59 """ 

60 try: 

61 return doImport(pluginName) 

62 except Exception as err: 

63 log.warning("Could not import plugin from %s, skipping.", pluginName) 

64 log.debug("Plugin import exception: %s\nTraceback:\n%s", err, 

65 "".join(traceback.format_tb(err.__traceback__))) 

66 return None 

67 

68 

69class LoaderCLI(click.MultiCommand, abc.ABC): 

70 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

71 subcommands at runtime.""" 

72 

73 def __init__(self, *args, **kwargs): 

74 super().__init__(*args, **kwargs) 

75 

76 @property 

77 @abc.abstractmethod 

78 def localCmdPkg(self): 

79 """localCmdPkg identifies the location of the commands that are in this 

80 package. `getLocalCommands` assumes that the commands can be found in 

81 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

82 should be overridden. 

83 

84 Returns 

85 ------- 

86 package : `str` 

87 The fully qualified location of this package. 

88 """ 

89 pass 

90 

91 def getLocalCommands(self): 

92 """Get the commands offered by the local package. This assumes that the 

93 commands can be found in `localCmdPkg.__all__`, if this is not the case 

94 then this function should be overridden. 

95 

96 Returns 

97 ------- 

98 commands : `defaultdict` [`str`, `list` [`str`]] 

99 The key is the command name. The value is a list of package(s) that 

100 contains the command. 

101 """ 

102 commandsLocation = _importPlugin(self.localCmdPkg) 

103 if commandsLocation is None: 

104 # _importPlugins logs an error, don't need to do it again here. 

105 return defaultdict(list) 

106 return defaultdict(list, {self._funcNameToCmdName(f): 

107 [self.localCmdPkg] for f in commandsLocation.__all__}) 

108 

109 def list_commands(self, ctx): 

110 """Used by Click to get all the commands that can be called by the 

111 butler command, it is used to generate the --help output. 

112 

113 Parameters 

114 ---------- 

115 ctx : `click.Context` 

116 The current Click context. 

117 

118 Returns 

119 ------- 

120 commands : `list` [`str`] 

121 The names of the commands that can be called by the butler command. 

122 """ 

123 self._setupLogging(ctx) 

124 commands = self._getCommands() 

125 self._raiseIfDuplicateCommands(commands) 

126 return sorted(commands) 

127 

128 def get_command(self, ctx, name): 

129 """Used by Click to get a single command for execution. 

130 

131 Parameters 

132 ---------- 

133 ctx : `click.Context` 

134 The current Click context. 

135 name : `str` 

136 The name of the command to return. 

137 

138 Returns 

139 ------- 

140 command : `click.Command` 

141 A Command that wraps a callable command function. 

142 """ 

143 self._setupLogging(ctx) 

144 commands = self._getCommands() 

145 if name not in commands: 

146 return None 

147 self._raiseIfDuplicateCommands(commands) 

148 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name)) 

149 

150 def _setupLogging(self, ctx): 

151 """Init the logging system and config it for the command. 

152 

153 Subcommands may further configure the log settings.""" 

154 if isinstance(ctx, click.Context): 

155 CliLog.initLog(longlog=ctx.params.get(long_log_option.name(), False), 

156 log_tty=ctx.params.get(log_tty_option.name(), True), 

157 log_file=ctx.params.get(log_file_option.name(), ()), 

158 log_label=ctx.params.get(log_label_option.name(), ())) 

159 if log_level_option.name() in ctx.params: 

160 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

161 else: 

162 # This works around a bug in sphinx-click, where it passes in the 

163 # click.MultiCommand instead of the context. 

164 # https://github.com/click-contrib/sphinx-click/issues/70 

165 CliLog.initLog(longlog=False) 

166 logging.debug("The passed-in context was not a click.Context, could not determine --long-log or " 

167 "--log-level values.") 

168 

169 @classmethod 

170 def getPluginList(cls): 

171 """Get the list of importable yaml files that contain cli data for this 

172 command. 

173 

174 Returns 

175 ------- 

176 `list` [`str`] 

177 The list of files that contain yaml data about a cli plugin. 

178 """ 

179 if not hasattr(cls, "pluginEnvVar"): 

180 return [] 

181 pluginModules = os.environ.get(cls.pluginEnvVar) 

182 if pluginModules: 

183 return [p for p in pluginModules.split(":") if p != ''] 

184 return [] 

185 

186 @classmethod 

187 def _funcNameToCmdName(cls, functionName): 

188 """Convert function name to the butler command name: change 

189 underscores, (used in functions) to dashes (used in commands), and 

190 change local-package command names that conflict with python keywords 

191 to a legal function name. 

192 """ 

193 return functionName.replace("_", "-") 

194 

195 @classmethod 

196 def _cmdNameToFuncName(cls, commandName): 

197 """Convert butler command name to function name: change dashes (used in 

198 commands) to underscores (used in functions), and for local-package 

199 commands names that conflict with python keywords, change the local, 

200 legal, function name to the command name.""" 

201 return commandName.replace("-", "_") 

202 

203 @staticmethod 

204 def _mergeCommandLists(a, b): 

205 """Combine two dicts whose keys are strings (command name) and values 

206 are list of string (the package(s) that provide the named command). 

207 

208 Parameters 

209 ---------- 

210 a : `defaultdict` [`str`, `list` [`str`]] 

211 The key is the command name. The value is a list of package(s) that 

212 contains the command. 

213 b : (same as a) 

214 

215 Returns 

216 ------- 

217 commands : `defaultdict` [`str`: [`str`]] 

218 For convenience, returns a extended with b. ('a' is modified in 

219 place.) 

220 """ 

221 for key, val in b.items(): 

222 a[key].extend(val) 

223 return a 

224 

225 @classmethod 

226 def _getPluginCommands(cls): 

227 """Get the commands offered by plugin packages. 

228 

229 Returns 

230 ------- 

231 commands : `defaultdict` [`str`, `list` [`str`]] 

232 The key is the command name. The value is a list of package(s) that 

233 contains the command. 

234 """ 

235 commands = defaultdict(list) 

236 for pluginName in cls.getPluginList(): 

237 try: 

238 with open(pluginName, "r") as resourceFile: 

239 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

240 except Exception as err: 

241 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

242 continue 

243 if "cmd" not in resources: 

244 log.warning("No commands found in %s, skipping.", pluginName) 

245 continue 

246 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

247 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

248 return commands 

249 

250 def _getCommands(self): 

251 """Get the commands offered by daf_butler and plugin packages. 

252 

253 Returns 

254 ------- 

255 commands : `defaultdict` [`str`, `list` [`str`]] 

256 The key is the command name. The value is a list of package(s) that 

257 contains the command. 

258 """ 

259 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

260 

261 @staticmethod 

262 def _raiseIfDuplicateCommands(commands): 

263 """If any provided command is offered by more than one package raise an 

264 exception. 

265 

266 Parameters 

267 ---------- 

268 commands : `defaultdict` [`str`, `list` [`str`]] 

269 The key is the command name. The value is a list of package(s) that 

270 contains the command. 

271 

272 Raises 

273 ------ 

274 click.ClickException 

275 Raised if a command is offered by more than one package, with an 

276 error message to be displayed to the user. 

277 """ 

278 

279 msg = "" 

280 for command, packages in commands.items(): 

281 if len(packages) > 1: 

282 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

283 if msg: 

284 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.") 

285 

286 

287class ButlerCLI(LoaderCLI): 

288 

289 localCmdPkg = "lsst.daf.butler.cli.cmd" 

290 

291 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

292 

293 @classmethod 

294 def _funcNameToCmdName(cls, functionName): 

295 # Docstring inherited from base class. 

296 

297 # The "import" command name and "butler_import" function name are 

298 # defined in cli/cmd/commands.py, and if those names are changed they 

299 # must be changed here as well. 

300 # It is expected that there will be very few butler command names that 

301 # need to be changed because of e.g. conflicts with python keywords (as 

302 # is done here and in _cmdNameToFuncName for the 'import' command). If 

303 # this becomes a common need then some way of doing this should be 

304 # invented that is better than hard coding the function names into 

305 # these conversion functions. An extension of the 'cli/resources.yaml' 

306 # file (as is currently used in obs_base) might be a good way to do it. 

307 if functionName == "butler_import": 

308 return "import" 

309 return super()._funcNameToCmdName(functionName) 

310 

311 @classmethod 

312 def _cmdNameToFuncName(cls, commandName): 

313 # Docstring inherited from base class. 

314 if commandName == "import": 

315 return "butler_import" 

316 return super()._cmdNameToFuncName(commandName) 

317 

318 

319@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

320@log_level_option() 

321@long_log_option() 

322@log_file_option() 

323@log_tty_option() 

324@log_label_option() 

325@ClickProgressHandler.option 

326def cli(log_level, long_log, log_file, log_tty, log_label, progress): 

327 # log_level is handled by get_command and list_commands, and is called in 

328 # one of those functions before this is called. long_log is handled by 

329 # setup_logging. 

330 pass 

331 

332 

333def main(): 

334 return cli()