Coverage for python/lsst/daf/butler/cli/butler.py: 35%

122 statements  

« prev     ^ index     » next       coverage.py v7.5.0, created at 2024-04-24 23:50 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23__all__ = ( 

24 "LoaderCLI", 

25 "ButlerCLI", 

26 "cli", 

27 "main", 

28) 

29 

30 

31import abc 

32import functools 

33import logging 

34import os 

35import traceback 

36from collections import defaultdict 

37 

38import click 

39import yaml 

40from lsst.utils import doImport 

41 

42from .cliLog import CliLog 

43from .opt import log_file_option, log_label_option, log_level_option, log_tty_option, long_log_option 

44from .progress import ClickProgressHandler 

45 

46log = logging.getLogger(__name__) 

47 

48 

49@functools.lru_cache 

50def _importPlugin(pluginName): 

51 """Import a plugin that contains Click commands. 

52 

53 Parameters 

54 ---------- 

55 pluginName : `str` 

56 An importable module whose __all__ parameter contains the commands 

57 that can be called. 

58 

59 Returns 

60 ------- 

61 An imported module or None 

62 The imported module, or None if the module could not be imported. 

63 

64 Notes 

65 ----- 

66 A cache is used in order to prevent repeated reports of failure 

67 to import a module that can be triggered by ``butler --help``. 

68 """ 

69 try: 

70 return doImport(pluginName) 

71 except Exception as err: 

72 log.warning("Could not import plugin from %s, skipping.", pluginName) 

73 log.debug( 

74 "Plugin import exception: %s\nTraceback:\n%s", 

75 err, 

76 "".join(traceback.format_tb(err.__traceback__)), 

77 ) 

78 return None 

79 

80 

81class LoaderCLI(click.MultiCommand, abc.ABC): 

82 """Extends `click.MultiCommand`, which dispatches to subcommands, to load 

83 subcommands at runtime.""" 

84 

85 def __init__(self, *args, **kwargs): 

86 super().__init__(*args, **kwargs) 

87 

88 @property 

89 @abc.abstractmethod 

90 def localCmdPkg(self): 

91 """localCmdPkg identifies the location of the commands that are in this 

92 package. `getLocalCommands` assumes that the commands can be found in 

93 `localCmdPkg.__all__`, if this is not the case then getLocalCommands 

94 should be overridden. 

95 

96 Returns 

97 ------- 

98 package : `str` 

99 The fully qualified location of this package. 

100 """ 

101 pass 

102 

103 def getLocalCommands(self): 

104 """Get the commands offered by the local package. This assumes that the 

105 commands can be found in `localCmdPkg.__all__`, if this is not the case 

106 then this function should be overridden. 

107 

108 Returns 

109 ------- 

110 commands : `defaultdict` [`str`, `list` [`str`]] 

111 The key is the command name. The value is a list of package(s) that 

112 contains the command. 

113 """ 

114 commandsLocation = _importPlugin(self.localCmdPkg) 

115 if commandsLocation is None: 

116 # _importPlugins logs an error, don't need to do it again here. 

117 return defaultdict(list) 

118 return defaultdict( 

119 list, {self._funcNameToCmdName(f): [self.localCmdPkg] for f in commandsLocation.__all__} 

120 ) 

121 

122 def list_commands(self, ctx): 

123 """Used by Click to get all the commands that can be called by the 

124 butler command, it is used to generate the --help output. 

125 

126 Parameters 

127 ---------- 

128 ctx : `click.Context` 

129 The current Click context. 

130 

131 Returns 

132 ------- 

133 commands : `list` [`str`] 

134 The names of the commands that can be called by the butler command. 

135 """ 

136 self._setupLogging(ctx) 

137 commands = self._getCommands() 

138 self._raiseIfDuplicateCommands(commands) 

139 return sorted(commands) 

140 

141 def get_command(self, ctx, name): 

142 """Used by Click to get a single command for execution. 

143 

144 Parameters 

145 ---------- 

146 ctx : `click.Context` 

147 The current Click context. 

148 name : `str` 

149 The name of the command to return. 

150 

151 Returns 

152 ------- 

153 command : `click.Command` 

154 A Command that wraps a callable command function. 

155 """ 

156 self._setupLogging(ctx) 

157 commands = self._getCommands() 

158 if name not in commands: 

159 return None 

160 self._raiseIfDuplicateCommands(commands) 

161 return _importPlugin(commands[name][0] + "." + self._cmdNameToFuncName(name)) 

162 

163 def _setupLogging(self, ctx): 

164 """Init the logging system and config it for the command. 

165 

166 Subcommands may further configure the log settings.""" 

167 if isinstance(ctx, click.Context): 

168 CliLog.initLog( 

169 longlog=ctx.params.get(long_log_option.name(), False), 

170 log_tty=ctx.params.get(log_tty_option.name(), True), 

171 log_file=ctx.params.get(log_file_option.name(), ()), 

172 log_label=ctx.params.get(log_label_option.name(), ()), 

173 ) 

174 if log_level_option.name() in ctx.params: 

175 CliLog.setLogLevels(ctx.params[log_level_option.name()]) 

176 else: 

177 # This works around a bug in sphinx-click, where it passes in the 

178 # click.MultiCommand instead of the context. 

179 # https://github.com/click-contrib/sphinx-click/issues/70 

180 CliLog.initLog(longlog=False) 

181 logging.debug( 

182 "The passed-in context was not a click.Context, could not determine --long-log or " 

183 "--log-level values." 

184 ) 

185 

186 @classmethod 

187 def getPluginList(cls): 

188 """Get the list of importable yaml files that contain cli data for this 

189 command. 

190 

191 Returns 

192 ------- 

193 `list` [`str`] 

194 The list of files that contain yaml data about a cli plugin. 

195 """ 

196 if not hasattr(cls, "pluginEnvVar"): 

197 return [] 

198 pluginModules = os.environ.get(cls.pluginEnvVar) 

199 if pluginModules: 

200 return [p for p in pluginModules.split(":") if p != ""] 

201 return [] 

202 

203 @classmethod 

204 def _funcNameToCmdName(cls, functionName): 

205 """Convert function name to the butler command name: change 

206 underscores, (used in functions) to dashes (used in commands), and 

207 change local-package command names that conflict with python keywords 

208 to a legal function name. 

209 """ 

210 return functionName.replace("_", "-") 

211 

212 @classmethod 

213 def _cmdNameToFuncName(cls, commandName): 

214 """Convert butler command name to function name: change dashes (used in 

215 commands) to underscores (used in functions), and for local-package 

216 commands names that conflict with python keywords, change the local, 

217 legal, function name to the command name.""" 

218 return commandName.replace("-", "_") 

219 

220 @staticmethod 

221 def _mergeCommandLists(a, b): 

222 """Combine two dicts whose keys are strings (command name) and values 

223 are list of string (the package(s) that provide the named command). 

224 

225 Parameters 

226 ---------- 

227 a : `defaultdict` [`str`, `list` [`str`]] 

228 The key is the command name. The value is a list of package(s) that 

229 contains the command. 

230 b : (same as a) 

231 

232 Returns 

233 ------- 

234 commands : `defaultdict` [`str`: [`str`]] 

235 For convenience, returns a extended with b. ('a' is modified in 

236 place.) 

237 """ 

238 for key, val in b.items(): 

239 a[key].extend(val) 

240 return a 

241 

242 @classmethod 

243 def _getPluginCommands(cls): 

244 """Get the commands offered by plugin packages. 

245 

246 Returns 

247 ------- 

248 commands : `defaultdict` [`str`, `list` [`str`]] 

249 The key is the command name. The value is a list of package(s) that 

250 contains the command. 

251 """ 

252 commands = defaultdict(list) 

253 for pluginName in cls.getPluginList(): 

254 try: 

255 with open(pluginName, "r") as resourceFile: 

256 resources = defaultdict(list, yaml.safe_load(resourceFile)) 

257 except Exception as err: 

258 log.warning("Error loading commands from %s, skipping. %s", pluginName, err) 

259 continue 

260 if "cmd" not in resources: 

261 log.warning("No commands found in %s, skipping.", pluginName) 

262 continue 

263 pluginCommands = {cmd: [resources["cmd"]["import"]] for cmd in resources["cmd"]["commands"]} 

264 cls._mergeCommandLists(commands, defaultdict(list, pluginCommands)) 

265 return commands 

266 

267 def _getCommands(self): 

268 """Get the commands offered by daf_butler and plugin packages. 

269 

270 Returns 

271 ------- 

272 commands : `defaultdict` [`str`, `list` [`str`]] 

273 The key is the command name. The value is a list of package(s) that 

274 contains the command. 

275 """ 

276 return self._mergeCommandLists(self.getLocalCommands(), self._getPluginCommands()) 

277 

278 @staticmethod 

279 def _raiseIfDuplicateCommands(commands): 

280 """If any provided command is offered by more than one package raise an 

281 exception. 

282 

283 Parameters 

284 ---------- 

285 commands : `defaultdict` [`str`, `list` [`str`]] 

286 The key is the command name. The value is a list of package(s) that 

287 contains the command. 

288 

289 Raises 

290 ------ 

291 click.ClickException 

292 Raised if a command is offered by more than one package, with an 

293 error message to be displayed to the user. 

294 """ 

295 

296 msg = "" 

297 for command, packages in commands.items(): 

298 if len(packages) > 1: 

299 msg += f"Command '{command}' exists in packages {', '.join(packages)}. " 

300 if msg: 

301 raise click.ClickException(message=msg + "Duplicate commands are not supported, aborting.") 

302 

303 

304class ButlerCLI(LoaderCLI): 

305 localCmdPkg = "lsst.daf.butler.cli.cmd" 

306 

307 pluginEnvVar = "DAF_BUTLER_PLUGINS" 

308 

309 @classmethod 

310 def _funcNameToCmdName(cls, functionName): 

311 # Docstring inherited from base class. 

312 

313 # The "import" command name and "butler_import" function name are 

314 # defined in cli/cmd/commands.py, and if those names are changed they 

315 # must be changed here as well. 

316 # It is expected that there will be very few butler command names that 

317 # need to be changed because of e.g. conflicts with python keywords (as 

318 # is done here and in _cmdNameToFuncName for the 'import' command). If 

319 # this becomes a common need then some way of doing this should be 

320 # invented that is better than hard coding the function names into 

321 # these conversion functions. An extension of the 'cli/resources.yaml' 

322 # file (as is currently used in obs_base) might be a good way to do it. 

323 if functionName == "butler_import": 

324 return "import" 

325 return super()._funcNameToCmdName(functionName) 

326 

327 @classmethod 

328 def _cmdNameToFuncName(cls, commandName): 

329 # Docstring inherited from base class. 

330 if commandName == "import": 

331 return "butler_import" 

332 return super()._cmdNameToFuncName(commandName) 

333 

334 

335@click.command(cls=ButlerCLI, context_settings=dict(help_option_names=["-h", "--help"])) 

336@log_level_option() 

337@long_log_option() 

338@log_file_option() 

339@log_tty_option() 

340@log_label_option() 

341@ClickProgressHandler.option 

342def cli(log_level, long_log, log_file, log_tty, log_label, progress): 

343 # log_level is handled by get_command and list_commands, and is called in 

344 # one of those functions before this is called. long_log is handled by 

345 # setup_logging. 

346 pass 

347 

348 

349def main(): 

350 return cli()