Coverage for python/lsst/daf/butler/cli/butler.py: 36%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

121 statements  

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import abc 

23import functools 

24import logging 

25import os 

26import traceback 

27from collections import defaultdict 

28 

29import click 

30import yaml 

31from lsst.utils import doImport 

32 

33from .cliLog import CliLog 

34from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

35from .progress import ClickProgressHandler 

36 

37log = logging.getLogger(__name__) 

38 

39 

40@functools.lru_cache 

41def _importPlugin(pluginName): 

42 """Import a plugin that contains Click commands. 

43 

44 Parameters 

45 ---------- 

46 pluginName : `str` 

47 An importable module whose __all__ parameter contains the commands 

48 that can be called. 

49 

50 Returns 

51 ------- 

52 An imported module or None 

53 The imported module, or None if the module could not be imported. 

54 

55 Notes 

56 ----- 

57 A cache is used in order to prevent repeated reports of failure 

58 to import a module that can be triggered by ``butler --help``. 

59 """ 

60 try: 

61 return doImport(pluginName) 

62 except Exception as err: 

63 log.warning("Could not import plugin from %s, skipping.", pluginName) 

64 log.debug( 

65 "Plugin import exception: %s\nTraceback:\n%s", 

66 err, 

67 "".join(traceback.format_tb(err.__traceback__)), 

68 ) 

69 return None 

70 

71 

72class LoaderCLI(click.MultiCommand, abc.ABC): 

73 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

74 subcommands at runtime.""" 

75 

76 def __init__(self, *args, **kwargs): 

77 super().__init__(*args, **kwargs) 

78 

79 @property 

80 @abc.abstractmethod 

81 def localCmdPkg(self): 

82 """localCmdPkg identifies the location of the commands that are in this 

83 package. `getLocalCommands` assumes that the commands can be found in 

84 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

85 should be overridden. 

86 

87 Returns 

88 ------- 

89 package : `str` 

90 The fully qualified location of this package. 

91 """ 

92 pass 

93 

94 def getLocalCommands(self): 

95 """Get the commands offered by the local package. This assumes that the 

96 commands can be found in `localCmdPkg.__all__`, if this is not the case 

97 then this function should be overridden. 

98 

99 Returns 

100 ------- 

101 commands : `defaultdict` [`str`, `list` [`str`]] 

102 The key is the command name. The value is a list of package(s) that 

103 contains the command. 

104 """ 

105 commandsLocation = _importPlugin(self.localCmdPkg) 

106 if commandsLocation is None: 

107 # _importPlugins logs an error, don't need to do it again here. 

108 return defaultdict(list) 

109 return defaultdict( 

110 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

111 ) 

112 

113 def list_commands(self, ctx): 

114 """Used by Click to get all the commands that can be called by the 

115 butler command, it is used to generate the --help output. 

116 

117 Parameters 

118 ---------- 

119 ctx : `click.Context` 

120 The current Click context. 

121 

122 Returns 

123 ------- 

124 commands : `list` [`str`] 

125 The names of the commands that can be called by the butler command. 

126 """ 

127 self._setupLogging(ctx) 

128 commands = self._getCommands() 

129 self._raiseIfDuplicateCommands(commands) 

130 return sorted(commands) 

131 

132 def get_command(self, ctx, name): 

133 """Used by Click to get a single command for execution. 

134 

135 Parameters 

136 ---------- 

137 ctx : `click.Context` 

138 The current Click context. 

139 name : `str` 

140 The name of the command to return. 

141 

142 Returns 

143 ------- 

144 command : `click.Command` 

145 A Command that wraps a callable command function. 

146 """ 

147 self._setupLogging(ctx) 

148 commands = self._getCommands() 

149 if name not in commands: 

150 return None 

151 self._raiseIfDuplicateCommands(commands) 

152 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name)) 

153 

154 def _setupLogging(self, ctx): 

155 """Init the logging system and config it for the command. 

156 

157 Subcommands may further configure the log settings.""" 

158 if isinstance(ctx, click.Context): 

159 CliLog.initLog( 

160 longlog=ctx.params.get(long_log_option.name(), False), 

161 log_tty=ctx.params.get(log_tty_option.name(), True), 

162 log_file=ctx.params.get(log_file_option.name(), ()), 

163 log_label=ctx.params.get(log_label_option.name(), ()), 

164 ) 

165 if log_level_option.name() in ctx.params: 

166 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

167 else: 

168 # This works around a bug in sphinx-click, where it passes in the 

169 # click.MultiCommand instead of the context. 

170 # https://github.com/click-contrib/sphinx-click/issues/70 

171 CliLog.initLog(longlog=False) 

172 logging.debug( 

173 "The passed-in context was not a click.Context, could not determine --long-log or " 

174 "--log-level values." 

175 ) 

176 

177 @classmethod 

178 def getPluginList(cls): 

179 """Get the list of importable yaml files that contain cli data for this 

180 command. 

181 

182 Returns 

183 ------- 

184 `list` [`str`] 

185 The list of files that contain yaml data about a cli plugin. 

186 """ 

187 if not hasattr(cls, "pluginEnvVar"): 

188 return [] 

189 pluginModules = os.environ.get(cls.pluginEnvVar) 

190 if pluginModules: 

191 return [p for p in pluginModules.split(":") if p != ""] 

192 return [] 

193 

194 @classmethod 

195 def _funcNameToCmdName(cls, functionName): 

196 """Convert function name to the butler command name: change 

197 underscores, (used in functions) to dashes (used in commands), and 

198 change local-package command names that conflict with python keywords 

199 to a legal function name. 

200 """ 

201 return functionName.replace("_", "-") 

202 

203 @classmethod 

204 def _cmdNameToFuncName(cls, commandName): 

205 """Convert butler command name to function name: change dashes (used in 

206 commands) to underscores (used in functions), and for local-package 

207 commands names that conflict with python keywords, change the local, 

208 legal, function name to the command name.""" 

209 return commandName.replace("-", "_") 

210 

211 @staticmethod 

212 def _mergeCommandLists(a, b): 

213 """Combine two dicts whose keys are strings (command name) and values 

214 are list of string (the package(s) that provide the named command). 

215 

216 Parameters 

217 ---------- 

218 a : `defaultdict` [`str`, `list` [`str`]] 

219 The key is the command name. The value is a list of package(s) that 

220 contains the command. 

221 b : (same as a) 

222 

223 Returns 

224 ------- 

225 commands : `defaultdict` [`str`: [`str`]] 

226 For convenience, returns a extended with b. ('a' is modified in 

227 place.) 

228 """ 

229 for key, val in b.items(): 

230 a[key].extend(val) 

231 return a 

232 

233 @classmethod 

234 def _getPluginCommands(cls): 

235 """Get the commands offered by plugin packages. 

236 

237 Returns 

238 ------- 

239 commands : `defaultdict` [`str`, `list` [`str`]] 

240 The key is the command name. The value is a list of package(s) that 

241 contains the command. 

242 """ 

243 commands = defaultdict(list) 

244 for pluginName in cls.getPluginList(): 

245 try: 

246 with open(pluginName, "r") as resourceFile: 

247 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

248 except Exception as err: 

249 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

250 continue 

251 if "cmd" not in resources: 

252 log.warning("No commands found in %s, skipping.", pluginName) 

253 continue 

254 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

255 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

256 return commands 

257 

258 def _getCommands(self): 

259 """Get the commands offered by daf_butler and plugin packages. 

260 

261 Returns 

262 ------- 

263 commands : `defaultdict` [`str`, `list` [`str`]] 

264 The key is the command name. The value is a list of package(s) that 

265 contains the command. 

266 """ 

267 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

268 

269 @staticmethod 

270 def _raiseIfDuplicateCommands(commands): 

271 """If any provided command is offered by more than one package raise an 

272 exception. 

273 

274 Parameters 

275 ---------- 

276 commands : `defaultdict` [`str`, `list` [`str`]] 

277 The key is the command name. The value is a list of package(s) that 

278 contains the command. 

279 

280 Raises 

281 ------ 

282 click.ClickException 

283 Raised if a command is offered by more than one package, with an 

284 error message to be displayed to the user. 

285 """ 

286 

287 msg = "" 

288 for command, packages in commands.items(): 

289 if len(packages) > 1: 

290 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

291 if msg: 

292 raise click.ClickException(msg + "Duplicate commands are not supported, aborting.") 

293 

294 

295class ButlerCLI(LoaderCLI): 

296 

297 localCmdPkg = "lsst.daf.butler.cli.cmd" 

298 

299 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

300 

301 @classmethod 

302 def _funcNameToCmdName(cls, functionName): 

303 # Docstring inherited from base class. 

304 

305 # The "import" command name and "butler_import" function name are 

306 # defined in cli/cmd/commands.py, and if those names are changed they 

307 # must be changed here as well. 

308 # It is expected that there will be very few butler command names that 

309 # need to be changed because of e.g. conflicts with python keywords (as 

310 # is done here and in _cmdNameToFuncName for the 'import' command). If 

311 # this becomes a common need then some way of doing this should be 

312 # invented that is better than hard coding the function names into 

313 # these conversion functions. An extension of the 'cli/resources.yaml' 

314 # file (as is currently used in obs_base) might be a good way to do it. 

315 if functionName == "butler_import": 

316 return "import" 

317 return super()._funcNameToCmdName(functionName) 

318 

319 @classmethod 

320 def _cmdNameToFuncName(cls, commandName): 

321 # Docstring inherited from base class. 

322 if commandName == "import": 

323 return "butler_import" 

324 return super()._cmdNameToFuncName(commandName) 

325 

326 

327@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

328@log_level_option() 

329@long_log_option() 

330@log_file_option() 

331@log_tty_option() 

332@log_label_option() 

333@ClickProgressHandler.option 

334def cli(log_level, long_log, log_file, log_tty, log_label, progress): 

335 # log_level is handled by get_command and list_commands, and is called in 

336 # one of those functions before this is called. long_log is handled by 

337 # setup_logging. 

338 pass 

339 

340 

341def main(): 

342 return cli()