Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import click.testing 

24from contextlib import contextmanager 

25import copy 

26from functools import partial 

27import io 

28import os 

29import textwrap 

30import traceback 

31from unittest.mock import MagicMock, patch 

32import uuid 

33import yaml 

34 

35from .cliLog import CliLog 

36from ..core.utils import iterable 

37 

38 

39# CLI_MOCK_ENV is set by some tests as an environment variable, it 

40# indicates to the cli_handle_exception function that instead of executing the 

41# command implementation function it should use the Mocker class for unit test 

42# verification. 

43mockEnvVarKey = "CLI_MOCK_ENV" 

44mockEnvVar = {mockEnvVarKey: "1"} 

45 

46# This is used as the metavar argument to Options that accept multiple string 

47# inputs, which may be comma-separarated. For example: 

48# --my-opt foo,bar --my-opt baz. 

49# Other arguments to the Option should include multiple=true and 

50# callback=split_kv. 

51typeStrAcceptsMultiple = "TEXT ..." 

52typeStrAcceptsSingle = "TEXT" 

53 

54# For parameters that support key-value inputs, this defines the separator 

55# for those inputs. 

56split_kv_separator = "=" 

57 

58 

59def textTypeStr(multiple): 

60 """Get the text type string for CLI help documentation. 

61 

62 Parameters 

63 ---------- 

64 multiple : `bool` 

65 True if multiple text values are allowed, False if only one value is 

66 allowed. 

67 

68 Returns 

69 ------- 

70 textTypeStr : `str` 

71 The type string to use. 

72 """ 

73 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

74 

75 

76class Mocker: 

77 

78 mock = MagicMock() 

79 

80 def __init__(self, *args, **kwargs): 

81 """Mocker is a helper class for unit tests. It can be imported and 

82 called and later imported again and call can be verified. 

83 

84 For convenience, constructor arguments are forwarded to the call 

85 function. 

86 """ 

87 self.__call__(*args, **kwargs) 

88 

89 def __call__(self, *args, **kwargs): 

90 """Creates a MagicMock and stores it in a static variable that can 

91 later be verified. 

92 """ 

93 Mocker.mock(*args, **kwargs) 

94 

95 @classmethod 

96 def reset(cls): 

97 cls.mock.reset_mock() 

98 

99 

100class LogCliRunner(click.testing.CliRunner): 

101 """A test runner to use when the logging system will be initialized by code 

102 under test, calls CliLog.resetLog(), which undoes any logging setup that 

103 was done with the CliLog interface. 

104 

105 lsst.log modules can not be set back to an uninitialized state (python 

106 logging modules can be set back to NOTSET), instead they are set to 

107 `CliLog.defaultLsstLogLevel`.""" 

108 

109 def invoke(self, *args, **kwargs): 

110 result = super().invoke(*args, **kwargs) 

111 CliLog.resetLog() 

112 return result 

113 

114 

115def clickResultMsg(result): 

116 """Get a standard assert message from a click result 

117 

118 Parameters 

119 ---------- 

120 result : click.Result 

121 The result object returned from click.testing.CliRunner.invoke 

122 

123 Returns 

124 ------- 

125 msg : `str` 

126 The message string. 

127 """ 

128 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

129 if result.exception: 

130 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

131 return msg 

132 

133 

134@contextmanager 

135def command_test_env(runner, commandModule, commandName): 

136 """A context manager that creates (and then cleans up) an environment that 

137 provides a CLI plugin command with the given name. 

138 

139 Parameters 

140 ---------- 

141 runner : click.testing.CliRunner 

142 The test runner to use to create the isolated filesystem. 

143 commandModule : `str` 

144 The importable module that the command can be imported from. 

145 commandName : `str` 

146 The name of the command being published to import. 

147 """ 

148 with runner.isolated_filesystem(): 

149 with open("resources.yaml", "w") as f: 

150 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

151 # Add a colon to the end of the path on the next line, this tests the 

152 # case where the lookup in LoaderCLI._getPluginList generates an empty 

153 # string in one of the list entries and verifies that the empty string 

154 # is properly stripped out. 

155 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

156 yield 

157 

158 

159def addArgumentHelp(doc, helpText): 

160 """Add a Click argument's help message to a function's documentation. 

161 

162 This is needed because click presents arguments in the order the argument 

163 decorators are applied to a function, top down. But, the evaluation of the 

164 decorators happens bottom up, so if arguments just append their help to the 

165 function's docstring, the argument descriptions appear in reverse order 

166 from the order they are applied in. 

167 

168 Parameters 

169 ---------- 

170 doc : `str` 

171 The function's docstring. 

172 helpText : `str` 

173 The argument's help string to be inserted into the function's 

174 docstring. 

175 

176 Returns 

177 ------- 

178 doc : `str` 

179 Updated function documentation. 

180 """ 

181 if doc is None: 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true

182 doc = helpText 

183 else: 

184 doclines = doc.splitlines() 

185 doclines.insert(1, helpText) 

186 doclines.insert(1, "\n") 

187 doc = "\n".join(doclines) 

188 return doc 

189 

190 

191def split_commas(context, param, values): 

192 """Process a tuple of values, where each value may contain comma-separated 

193 values, and return a single list of all the passed-in values. 

194 

195 This function can be passed to the 'callback' argument of a click.option to 

196 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). 

197 

198 Parameters 

199 ---------- 

200 context : `click.Context` or `None` 

201 The current execution context. Unused, but Click always passes it to 

202 callbacks. 

203 param : `click.core.Option` or `None` 

204 The parameter being handled. Unused, but Click always passes it to 

205 callbacks. 

206 values : [`str`] 

207 All the values passed for this option. Strings may contain commas, 

208 which will be treated as delimiters for separate values. 

209 

210 Returns 

211 ------- 

212 list of string 

213 The passed in values separated by commas and combined into a single 

214 list. 

215 """ 

216 if values is None: 

217 return values 

218 valueList = [] 

219 for value in iterable(values): 

220 valueList.extend(value.split(",")) 

221 return tuple(valueList) 

222 

223 

224def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=", 

225 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False): 

226 """Process a tuple of values that are key-value pairs separated by a given 

227 separator. Multiple pairs may be comma separated. Return a dictionary of 

228 all the passed-in values. 

229 

230 This function can be passed to the 'callback' argument of a click.option to 

231 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

232 

233 Parameters 

234 ---------- 

235 context : `click.Context` or `None` 

236 The current execution context. Unused, but Click always passes it to 

237 callbacks. 

238 param : `click.core.Option` or `None` 

239 The parameter being handled. Unused, but Click always passes it to 

240 callbacks. 

241 values : [`str`] 

242 All the values passed for this option. Strings may contain commas, 

243 which will be treated as delimiters for separate values. 

244 choice : `click.Choice`, optional 

245 If provided, verify each value is a valid choice using the provided 

246 `click.Choice` instance. If None, no verification will be done. By 

247 default None 

248 multiple : `bool`, optional 

249 If true, the value may contain multiple comma-separated values. By 

250 default True. 

251 normalize : `bool`, optional 

252 If True and `choice.case_sensitive == False`, normalize the string the 

253 user provided to match the choice's case. By default False. 

254 separator : str, optional 

255 The character that separates key-value pairs. May not be a comma or an 

256 empty space (for space separators use Click's default implementation 

257 for tuples; `type=(str, str)`). By default "=". 

258 unseparated_okay : `bool`, optional 

259 If True, allow values that do not have a separator. They will be 

260 returned in the values dict as a tuple of values in the key '', that 

261 is: `values[''] = (unseparated_values, )`. By default False. 

262 return_type : `type`, must be `dict` or `tuple` 

263 The type of the value that should be returned. 

264 If `dict` then the returned object will be a dict, for each item in 

265 values, the value to the left of the separator will be the key and the 

266 value to the right of the separator will be the value. 

267 If `tuple` then the returned object will be a tuple. Each item in the 

268 tuple will be 2-item tuple, the first item will be the value to the 

269 left of the separator and the second item will be the value to the 

270 right. By default `dict`. 

271 default_key : `Any` 

272 The key to use if a value is passed that is not a key-value pair. 

273 (Passing values that are not key-value pairs requires 

274 ``unseparated_okay`` to be `True`.) 

275 reverse_kv : bool 

276 If true then for each item in values, the value to the left of the 

277 separator is treated as the value and the value to the right of the 

278 separator is treated as the key. By default False. 

279 

280 Returns 

281 ------- 

282 values : `dict` [`str`, `str`] 

283 The passed-in values in dict form. 

284 

285 Raises 

286 ------ 

287 `click.ClickException` 

288 Raised if the separator is not found in an entry, or if duplicate keys 

289 are encountered. 

290 """ 

291 

292 def norm(val): 

293 """If `normalize` is True and `choice` is not `None`, find the value 

294 in the available choices and return the value as spelled in the 

295 choices. 

296 

297 Assumes that val exists in choices; `split_kv` uses the `choice` 

298 instance to verify val is a valid choice. 

299 """ 

300 if normalize and choice is not None: 

301 v = val.casefold() 

302 for opt in choice.choices: 

303 if opt.casefold() == v: 

304 return opt 

305 return val 

306 

307 class RetDict: 

308 

309 def __init__(self): 

310 self.ret = {} 

311 

312 def add(self, key, val): 

313 if reverse_kv: 

314 key, val = val, key 

315 if key in self.ret: 

316 raise click.ClickException(f"Duplicate entries for '{k}' in '{values}'") 

317 self.ret[key] = val 

318 

319 def get(self): 

320 return self.ret 

321 

322 class RetTuple: 

323 

324 def __init__(self): 

325 self.ret = [] 

326 

327 def add(self, key, val): 

328 if reverse_kv: 

329 key, val = val, key 

330 self.ret.append((key, val)) 

331 

332 def get(self): 

333 return tuple(self.ret) 

334 

335 if separator in (",", " "): 

336 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

337 vals = values # preserve the original argument for error reporting below. 

338 if return_type is dict: 

339 ret = RetDict() 

340 elif return_type is tuple: 

341 ret = RetTuple() 

342 else: 

343 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.") 

344 if multiple: 

345 vals = split_commas(context, param, vals) 

346 for val in iterable(vals): 

347 if unseparated_okay and separator not in val: 

348 if choice is not None: 

349 choice(val) # will raise if val is an invalid choice 

350 ret.add(default_key, norm(val)) 

351 else: 

352 try: 

353 k, v = val.split(separator) 

354 if choice is not None: 

355 choice(v) # will raise if val is an invalid choice 

356 except ValueError: 

357 raise click.ClickException( 

358 f"Could not parse key-value pair '{val}' using separator '{separator}', " 

359 f"with multiple values {'allowed' if multiple else 'not allowed'}.") 

360 ret.add(k, norm(v)) 

361 return ret.get() 

362 

363 

364def to_upper(context, param, value): 

365 """Convert a value to upper case. 

366 

367 Parameters 

368 ---------- 

369 context : click.Context 

370 

371 values : string 

372 The value to be converted. 

373 

374 Returns 

375 ------- 

376 string 

377 A copy of the passed-in value, converted to upper case. 

378 """ 

379 return value.upper() 

380 

381 

382def unwrap(val): 

383 """Remove newlines and leading whitespace from a multi-line string with 

384 a consistent indentation level. 

385 

386 The first line of the string may be only a newline or may contain text 

387 followed by a newline, either is ok. After the first line, each line must 

388 begin with a consistant amount of whitespace. So, content of a 

389 triple-quoted string may begin immediately after the quotes, or the string 

390 may start with a newline. Each line after that must be the same amount of 

391 indentation/whitespace followed by text and a newline. The last line may 

392 end with a new line but is not required to do so. 

393 

394 Parameters 

395 ---------- 

396 val : `str` 

397 The string to change. 

398 

399 Returns 

400 ------- 

401 strippedString : `str` 

402 The string with newlines, indentation, and leading and trailing 

403 whitespace removed. 

404 """ 

405 def splitSection(val): 

406 if not val.startswith("\n"): 406 ↛ 410line 406 didn't jump to line 410, because the condition on line 406 was never false

407 firstLine, _, val = val.partition("\n") 

408 firstLine += " " 

409 else: 

410 firstLine = "" 

411 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

412 

413 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

414 

415 

416def cli_handle_exception(func, *args, **kwargs): 

417 """Wrap a function call in an exception handler that raises a 

418 ClickException if there is an Exception. 

419 

420 Also provides support for unit testing by testing for an environment 

421 variable, and if it is present prints the function name, args, and kwargs 

422 to stdout so they can be read and verified by the unit test code. 

423 

424 Parameters 

425 ---------- 

426 func : function 

427 A function to be called and exceptions handled. Will pass args & kwargs 

428 to the function. 

429 

430 Returns 

431 ------- 

432 The result of calling func. 

433 

434 Raises 

435 ------ 

436 click.ClickException 

437 An exception to be handled by the Click CLI tool. 

438 """ 

439 if mockEnvVarKey in os.environ: 

440 Mocker(*args, **kwargs) 

441 return 

442 try: 

443 return func(*args, **kwargs) 

444 except Exception: 

445 msg = io.StringIO() 

446 msg.write("An error occurred during command execution:\n") 

447 traceback.print_exc(file=msg) 

448 raise click.ClickException(msg.getvalue()) 

449 

450 

451class option_section: # noqa: N801 

452 """Decorator to add a section label between options in the help text of a 

453 command. 

454 

455 Parameters 

456 ---------- 

457 sectionText : `str` 

458 The text to print in the section identifier. 

459 """ 

460 

461 def __init__(self, sectionText): 

462 self.sectionText = "\n" + sectionText 

463 

464 def __call__(self, f): 

465 # Generate a parameter declaration that will be unique for this 

466 # section. 

467 return click.option(f"--option-section-{str(uuid.uuid4())}", 

468 sectionText=self.sectionText, 

469 cls=OptionSection)(f) 

470 

471 

472class MWPath(click.Path): 

473 """Overrides click.Path to implement file-does-not-exist checking. 

474 

475 Changes the definition of ``exists` so that `True` indicates the location 

476 (file or directory) must exist, `False` indicates the location must *not* 

477 exist, and `None` indicates that the file may exist or not. The standard 

478 definition for the `click.Path` ``exists`` parameter is that for `True` a 

479 location must exist, but `False` means it is not required to exist (not 

480 that it is required to not exist). 

481 

482 Parameters 

483 ---------- 

484 exists : `True`, `False`, or `None` 

485 If `True`, the location (file or directory) indicated by the caller 

486 must exist. If `False` the location must not exist. If `None`, the 

487 location may exist or not. 

488 

489 For other parameters see `click.Path`. 

490 """ 

491 

492 def __init__(self, exists=None, file_okay=True, dir_okay=True, 

493 writable=False, readable=True, resolve_path=False, 

494 allow_dash=False, path_type=None): 

495 self.mustNotExist = exists is False 

496 if exists is None: 

497 exists = False 

498 super().__init__(exists, file_okay, dir_okay, writable, readable, 

499 resolve_path, allow_dash, path_type) 

500 

501 def convert(self, value, param, ctx): 

502 """Called by click.ParamType to "convert values through types". 

503 `click.Path` uses this step to verify Path conditions.""" 

504 if self.mustNotExist and os.path.exists(value): 

505 self.fail(f'{self.path_type} "{value}" should not exist.') 

506 return super().convert(value, param, ctx) 

507 

508 

509class MWOption(click.Option): 

510 """Overrides click.Option with desired behaviors.""" 

511 

512 def make_metavar(self): 

513 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

514 help menu. Adds a space and an elipsis after the metavar name if 

515 the option accepts multiple inputs, otherwise defers to the base 

516 implementation. 

517 

518 By default click does not add an elipsis when multiple is True and 

519 nargs is 1. And when nargs does not equal 1 click adds an elipsis 

520 without a space between the metavar and the elipsis, but we prefer a 

521 space between. 

522 

523 Does not get called for some option types (e.g. flag) so metavar 

524 transformation that must apply to all types should be applied in 

525 get_help_record. 

526 """ 

527 metavar = super().make_metavar() 

528 if self.multiple and self.nargs == 1: 

529 metavar += " ..." 

530 elif self.nargs != 1: 

531 metavar = f"{metavar[:-3]} ..." 

532 return metavar 

533 

534 

535class MWArgument(click.Argument): 

536 """Overrides click.Argument with desired behaviors.""" 

537 

538 def make_metavar(self): 

539 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

540 help menu. Always adds a space and an elipsis (' ...') after the 

541 metavar name if the option accepts multiple inputs. 

542 

543 By default click adds an elipsis without a space between the metavar 

544 and the elipsis, but we prefer a space between. 

545 

546 Returns 

547 ------- 

548 metavar : `str` 

549 The metavar value. 

550 """ 

551 metavar = super().make_metavar() 

552 if self.nargs != 1: 

553 metavar = f"{metavar[:-3]} ..." 

554 return metavar 

555 

556 

557class OptionSection(MWOption): 

558 """Implements an Option that prints a section label in the help text and 

559 does not pass any value to the command function. 

560 

561 This class does a bit of hackery to add a section label to a click command 

562 help output: first, `expose_value` is set to `False` so that no value is 

563 passed to the command function. Second, this class overrides 

564 `click.Option.get_help_record` to return the section label string without 

565 any prefix so that it stands out as a section label. 

566 

567 The intention for this implementation is to do minimally invasive overrides 

568 of the click classes so as to be robust and easy to fix if the click 

569 internals change. 

570 

571 Parameters 

572 ---------- 

573 sectionName : `str` 

574 The parameter declaration for this option. It is not shown to the user, 

575 it must be unique within the command. If using the `section` decorator 

576 to add a section to a command's options, the section name is 

577 auto-generated. 

578 sectionText : `str` 

579 The text to print in the section identifier. 

580 """ 

581 

582 def __init__(self, sectionName, sectionText): 

583 super().__init__(sectionName, expose_value=False) 

584 self.sectionText = sectionText 

585 

586 def get_help_record(self, ctx): 

587 return (self.sectionText, "") 

588 

589 

590class MWOptionDecorator: 

591 """Wraps the click.option decorator to enable shared options to be declared 

592 and allows inspection of the shared option. 

593 """ 

594 

595 def __init__(self, *param_decls, **kwargs): 

596 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), 

597 **kwargs) 

598 opt = click.Option(param_decls, **kwargs) 

599 self._name = opt.name 

600 self._opts = opt.opts 

601 

602 def name(self): 

603 """Get the name that will be passed to the command function for this 

604 option.""" 

605 return self._name 

606 

607 def opts(self): 

608 """Get the flags that will be used for this option on the command 

609 line.""" 

610 return self._opts 

611 

612 def __call__(self, *args, **kwargs): 

613 return self.partialOpt(*args, **kwargs) 

614 

615 

616class MWArgumentDecorator: 

617 """Wraps the click.argument decorator to enable shared arguments to be 

618 declared. """ 

619 

620 def __init__(self, *param_decls, **kwargs): 

621 self._helpText = kwargs.pop("help", None) 

622 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

623 

624 def __call__(self, *args, help=None, **kwargs): 

625 def decorator(f): 

626 if help is not None: 

627 self._helpText = help 

628 if self._helpText: 628 ↛ 630line 628 didn't jump to line 630, because the condition on line 628 was never false

629 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

630 return self.partialArg(*args, **kwargs)(f) 

631 return decorator 

632 

633 

634class MWCommand(click.Command): 

635 """Command subclass that stores a copy of the args list for use by the 

636 command.""" 

637 

638 def parse_args(self, ctx, args): 

639 MWCtxObj.getFrom(ctx).args = copy.copy(args) 

640 super().parse_args(ctx, args) 

641 

642 

643class MWCtxObj(): 

644 """Helper object for managing the `click.Context.obj` parameter, allows 

645 obj data to be managed in a consistent way. 

646 

647 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

648 initialize the obj if needed and return a new or existing MWCtxObj. 

649 

650 Attributes 

651 ---------- 

652 args : `list` [`str`] 

653 The list of arguments (argument values, option flags, and option 

654 values), split using whitespace, that were passed in on the command 

655 line for the subcommand represented by the parent context object. 

656 """ 

657 

658 def __init__(self): 

659 

660 self.args = None 

661 

662 @staticmethod 

663 def getFrom(ctx): 

664 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the 

665 new or already existing MWCtxObj.""" 

666 if ctx.obj is not None: 

667 return ctx.obj 

668 ctx.obj = MWCtxObj() 

669 return ctx.obj