Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import click.testing 

24from contextlib import contextmanager 

25import copy 

26from functools import partial 

27import io 

28import os 

29import textwrap 

30import traceback 

31from unittest.mock import MagicMock, patch 

32import uuid 

33import yaml 

34import logging 

35 

36from .cliLog import CliLog 

37from ..core.utils import iterable 

38from ..core.config import Config 

39 

40log = logging.getLogger(__name__) 

41 

42# CLI_MOCK_ENV is set by some tests as an environment variable, it 

43# indicates to the cli_handle_exception function that instead of executing the 

44# command implementation function it should use the Mocker class for unit test 

45# verification. 

46mockEnvVarKey = "CLI_MOCK_ENV" 

47mockEnvVar = {mockEnvVarKey: "1"} 

48 

49# This is used as the metavar argument to Options that accept multiple string 

50# inputs, which may be comma-separarated. For example: 

51# --my-opt foo,bar --my-opt baz. 

52# Other arguments to the Option should include multiple=true and 

53# callback=split_kv. 

54typeStrAcceptsMultiple = "TEXT ..." 

55typeStrAcceptsSingle = "TEXT" 

56 

57# For parameters that support key-value inputs, this defines the separator 

58# for those inputs. 

59split_kv_separator = "=" 

60 

61 

62def textTypeStr(multiple): 

63 """Get the text type string for CLI help documentation. 

64 

65 Parameters 

66 ---------- 

67 multiple : `bool` 

68 True if multiple text values are allowed, False if only one value is 

69 allowed. 

70 

71 Returns 

72 ------- 

73 textTypeStr : `str` 

74 The type string to use. 

75 """ 

76 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

77 

78 

79class Mocker: 

80 

81 mock = MagicMock() 

82 

83 def __init__(self, *args, **kwargs): 

84 """Mocker is a helper class for unit tests. It can be imported and 

85 called and later imported again and call can be verified. 

86 

87 For convenience, constructor arguments are forwarded to the call 

88 function. 

89 """ 

90 self.__call__(*args, **kwargs) 

91 

92 def __call__(self, *args, **kwargs): 

93 """Creates a MagicMock and stores it in a static variable that can 

94 later be verified. 

95 """ 

96 Mocker.mock(*args, **kwargs) 

97 

98 @classmethod 

99 def reset(cls): 

100 cls.mock.reset_mock() 

101 

102 

103class LogCliRunner(click.testing.CliRunner): 

104 """A test runner to use when the logging system will be initialized by code 

105 under test, calls CliLog.resetLog(), which undoes any logging setup that 

106 was done with the CliLog interface. 

107 

108 lsst.log modules can not be set back to an uninitialized state (python 

109 logging modules can be set back to NOTSET), instead they are set to 

110 `CliLog.defaultLsstLogLevel`.""" 

111 

112 def invoke(self, *args, **kwargs): 

113 result = super().invoke(*args, **kwargs) 

114 CliLog.resetLog() 

115 return result 

116 

117 

118def clickResultMsg(result): 

119 """Get a standard assert message from a click result 

120 

121 Parameters 

122 ---------- 

123 result : click.Result 

124 The result object returned from click.testing.CliRunner.invoke 

125 

126 Returns 

127 ------- 

128 msg : `str` 

129 The message string. 

130 """ 

131 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

132 if result.exception: 

133 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

134 return msg 

135 

136 

137@contextmanager 

138def command_test_env(runner, commandModule, commandName): 

139 """A context manager that creates (and then cleans up) an environment that 

140 provides a CLI plugin command with the given name. 

141 

142 Parameters 

143 ---------- 

144 runner : click.testing.CliRunner 

145 The test runner to use to create the isolated filesystem. 

146 commandModule : `str` 

147 The importable module that the command can be imported from. 

148 commandName : `str` 

149 The name of the command being published to import. 

150 """ 

151 with runner.isolated_filesystem(): 

152 with open("resources.yaml", "w") as f: 

153 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

154 # Add a colon to the end of the path on the next line, this tests the 

155 # case where the lookup in LoaderCLI._getPluginList generates an empty 

156 # string in one of the list entries and verifies that the empty string 

157 # is properly stripped out. 

158 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

159 yield 

160 

161 

162def addArgumentHelp(doc, helpText): 

163 """Add a Click argument's help message to a function's documentation. 

164 

165 This is needed because click presents arguments in the order the argument 

166 decorators are applied to a function, top down. But, the evaluation of the 

167 decorators happens bottom up, so if arguments just append their help to the 

168 function's docstring, the argument descriptions appear in reverse order 

169 from the order they are applied in. 

170 

171 Parameters 

172 ---------- 

173 doc : `str` 

174 The function's docstring. 

175 helpText : `str` 

176 The argument's help string to be inserted into the function's 

177 docstring. 

178 

179 Returns 

180 ------- 

181 doc : `str` 

182 Updated function documentation. 

183 """ 

184 if doc is None: 184 ↛ 185line 184 didn't jump to line 185, because the condition on line 184 was never true

185 doc = helpText 

186 else: 

187 doclines = doc.splitlines() 

188 doclines.insert(1, helpText) 

189 doclines.insert(1, "\n") 

190 doc = "\n".join(doclines) 

191 return doc 

192 

193 

194def split_commas(context, param, values): 

195 """Process a tuple of values, where each value may contain comma-separated 

196 values, and return a single list of all the passed-in values. 

197 

198 This function can be passed to the 'callback' argument of a click.option to 

199 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). 

200 

201 Parameters 

202 ---------- 

203 context : `click.Context` or `None` 

204 The current execution context. Unused, but Click always passes it to 

205 callbacks. 

206 param : `click.core.Option` or `None` 

207 The parameter being handled. Unused, but Click always passes it to 

208 callbacks. 

209 values : [`str`] 

210 All the values passed for this option. Strings may contain commas, 

211 which will be treated as delimiters for separate values. 

212 

213 Returns 

214 ------- 

215 list of string 

216 The passed in values separated by commas and combined into a single 

217 list. 

218 """ 

219 if values is None: 

220 return values 

221 valueList = [] 

222 for value in iterable(values): 

223 valueList.extend(value.split(",")) 

224 return tuple(valueList) 

225 

226 

227def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=", 

228 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False): 

229 """Process a tuple of values that are key-value pairs separated by a given 

230 separator. Multiple pairs may be comma separated. Return a dictionary of 

231 all the passed-in values. 

232 

233 This function can be passed to the 'callback' argument of a click.option to 

234 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

235 

236 Parameters 

237 ---------- 

238 context : `click.Context` or `None` 

239 The current execution context. Unused, but Click always passes it to 

240 callbacks. 

241 param : `click.core.Option` or `None` 

242 The parameter being handled. Unused, but Click always passes it to 

243 callbacks. 

244 values : [`str`] 

245 All the values passed for this option. Strings may contain commas, 

246 which will be treated as delimiters for separate values. 

247 choice : `click.Choice`, optional 

248 If provided, verify each value is a valid choice using the provided 

249 `click.Choice` instance. If None, no verification will be done. By 

250 default None 

251 multiple : `bool`, optional 

252 If true, the value may contain multiple comma-separated values. By 

253 default True. 

254 normalize : `bool`, optional 

255 If True and `choice.case_sensitive == False`, normalize the string the 

256 user provided to match the choice's case. By default False. 

257 separator : str, optional 

258 The character that separates key-value pairs. May not be a comma or an 

259 empty space (for space separators use Click's default implementation 

260 for tuples; `type=(str, str)`). By default "=". 

261 unseparated_okay : `bool`, optional 

262 If True, allow values that do not have a separator. They will be 

263 returned in the values dict as a tuple of values in the key '', that 

264 is: `values[''] = (unseparated_values, )`. By default False. 

265 return_type : `type`, must be `dict` or `tuple` 

266 The type of the value that should be returned. 

267 If `dict` then the returned object will be a dict, for each item in 

268 values, the value to the left of the separator will be the key and the 

269 value to the right of the separator will be the value. 

270 If `tuple` then the returned object will be a tuple. Each item in the 

271 tuple will be 2-item tuple, the first item will be the value to the 

272 left of the separator and the second item will be the value to the 

273 right. By default `dict`. 

274 default_key : `Any` 

275 The key to use if a value is passed that is not a key-value pair. 

276 (Passing values that are not key-value pairs requires 

277 ``unseparated_okay`` to be `True`.) 

278 reverse_kv : bool 

279 If true then for each item in values, the value to the left of the 

280 separator is treated as the value and the value to the right of the 

281 separator is treated as the key. By default False. 

282 

283 Returns 

284 ------- 

285 values : `dict` [`str`, `str`] 

286 The passed-in values in dict form. 

287 

288 Raises 

289 ------ 

290 `click.ClickException` 

291 Raised if the separator is not found in an entry, or if duplicate keys 

292 are encountered. 

293 """ 

294 

295 def norm(val): 

296 """If `normalize` is True and `choice` is not `None`, find the value 

297 in the available choices and return the value as spelled in the 

298 choices. 

299 

300 Assumes that val exists in choices; `split_kv` uses the `choice` 

301 instance to verify val is a valid choice. 

302 """ 

303 if normalize and choice is not None: 

304 v = val.casefold() 

305 for opt in choice.choices: 

306 if opt.casefold() == v: 

307 return opt 

308 return val 

309 

310 class RetDict: 

311 

312 def __init__(self): 

313 self.ret = {} 

314 

315 def add(self, key, val): 

316 if reverse_kv: 

317 key, val = val, key 

318 if key in self.ret: 

319 raise click.ClickException(f"Duplicate entries for '{k}' in '{values}'") 

320 self.ret[key] = val 

321 

322 def get(self): 

323 return self.ret 

324 

325 class RetTuple: 

326 

327 def __init__(self): 

328 self.ret = [] 

329 

330 def add(self, key, val): 

331 if reverse_kv: 

332 key, val = val, key 

333 self.ret.append((key, val)) 

334 

335 def get(self): 

336 return tuple(self.ret) 

337 

338 if separator in (",", " "): 

339 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

340 vals = values # preserve the original argument for error reporting below. 

341 if return_type is dict: 

342 ret = RetDict() 

343 elif return_type is tuple: 

344 ret = RetTuple() 

345 else: 

346 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.") 

347 if multiple: 

348 vals = split_commas(context, param, vals) 

349 for val in iterable(vals): 

350 if unseparated_okay and separator not in val: 

351 if choice is not None: 

352 choice(val) # will raise if val is an invalid choice 

353 ret.add(default_key, norm(val)) 

354 else: 

355 try: 

356 k, v = val.split(separator) 

357 if choice is not None: 

358 choice(v) # will raise if val is an invalid choice 

359 except ValueError: 

360 raise click.ClickException( 

361 f"Could not parse key-value pair '{val}' using separator '{separator}', " 

362 f"with multiple values {'allowed' if multiple else 'not allowed'}.") 

363 ret.add(k, norm(v)) 

364 return ret.get() 

365 

366 

367def to_upper(context, param, value): 

368 """Convert a value to upper case. 

369 

370 Parameters 

371 ---------- 

372 context : click.Context 

373 

374 values : string 

375 The value to be converted. 

376 

377 Returns 

378 ------- 

379 string 

380 A copy of the passed-in value, converted to upper case. 

381 """ 

382 return value.upper() 

383 

384 

385def unwrap(val): 

386 """Remove newlines and leading whitespace from a multi-line string with 

387 a consistent indentation level. 

388 

389 The first line of the string may be only a newline or may contain text 

390 followed by a newline, either is ok. After the first line, each line must 

391 begin with a consistant amount of whitespace. So, content of a 

392 triple-quoted string may begin immediately after the quotes, or the string 

393 may start with a newline. Each line after that must be the same amount of 

394 indentation/whitespace followed by text and a newline. The last line may 

395 end with a new line but is not required to do so. 

396 

397 Parameters 

398 ---------- 

399 val : `str` 

400 The string to change. 

401 

402 Returns 

403 ------- 

404 strippedString : `str` 

405 The string with newlines, indentation, and leading and trailing 

406 whitespace removed. 

407 """ 

408 def splitSection(val): 

409 if not val.startswith("\n"): 409 ↛ 413line 409 didn't jump to line 413, because the condition on line 409 was never false

410 firstLine, _, val = val.partition("\n") 

411 firstLine += " " 

412 else: 

413 firstLine = "" 

414 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

415 

416 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

417 

418 

419def cli_handle_exception(func, *args, **kwargs): 

420 """Wrap a function call in an exception handler that raises a 

421 ClickException if there is an Exception. 

422 

423 Also provides support for unit testing by testing for an environment 

424 variable, and if it is present prints the function name, args, and kwargs 

425 to stdout so they can be read and verified by the unit test code. 

426 

427 Parameters 

428 ---------- 

429 func : function 

430 A function to be called and exceptions handled. Will pass args & kwargs 

431 to the function. 

432 

433 Returns 

434 ------- 

435 The result of calling func. 

436 

437 Raises 

438 ------ 

439 click.ClickException 

440 An exception to be handled by the Click CLI tool. 

441 """ 

442 if mockEnvVarKey in os.environ: 

443 Mocker(*args, **kwargs) 

444 return 

445 try: 

446 return func(*args, **kwargs) 

447 except Exception: 

448 msg = io.StringIO() 

449 msg.write("An error occurred during command execution:\n") 

450 traceback.print_exc(file=msg) 

451 raise click.ClickException(msg.getvalue()) 

452 

453 

454class option_section: # noqa: N801 

455 """Decorator to add a section label between options in the help text of a 

456 command. 

457 

458 Parameters 

459 ---------- 

460 sectionText : `str` 

461 The text to print in the section identifier. 

462 """ 

463 

464 def __init__(self, sectionText): 

465 self.sectionText = "\n" + sectionText 

466 

467 def __call__(self, f): 

468 # Generate a parameter declaration that will be unique for this 

469 # section. 

470 return click.option(f"--option-section-{str(uuid.uuid4())}", 

471 sectionText=self.sectionText, 

472 cls=OptionSection)(f) 

473 

474 

475class MWPath(click.Path): 

476 """Overrides click.Path to implement file-does-not-exist checking. 

477 

478 Changes the definition of ``exists` so that `True` indicates the location 

479 (file or directory) must exist, `False` indicates the location must *not* 

480 exist, and `None` indicates that the file may exist or not. The standard 

481 definition for the `click.Path` ``exists`` parameter is that for `True` a 

482 location must exist, but `False` means it is not required to exist (not 

483 that it is required to not exist). 

484 

485 Parameters 

486 ---------- 

487 exists : `True`, `False`, or `None` 

488 If `True`, the location (file or directory) indicated by the caller 

489 must exist. If `False` the location must not exist. If `None`, the 

490 location may exist or not. 

491 

492 For other parameters see `click.Path`. 

493 """ 

494 

495 def __init__(self, exists=None, file_okay=True, dir_okay=True, 

496 writable=False, readable=True, resolve_path=False, 

497 allow_dash=False, path_type=None): 

498 self.mustNotExist = exists is False 

499 if exists is None: 

500 exists = False 

501 super().__init__(exists, file_okay, dir_okay, writable, readable, 

502 resolve_path, allow_dash, path_type) 

503 

504 def convert(self, value, param, ctx): 

505 """Called by click.ParamType to "convert values through types". 

506 `click.Path` uses this step to verify Path conditions.""" 

507 if self.mustNotExist and os.path.exists(value): 

508 self.fail(f'{self.path_type} "{value}" should not exist.') 

509 return super().convert(value, param, ctx) 

510 

511 

512class MWOption(click.Option): 

513 """Overrides click.Option with desired behaviors.""" 

514 

515 def make_metavar(self): 

516 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

517 help menu. Adds a space and an elipsis after the metavar name if 

518 the option accepts multiple inputs, otherwise defers to the base 

519 implementation. 

520 

521 By default click does not add an elipsis when multiple is True and 

522 nargs is 1. And when nargs does not equal 1 click adds an elipsis 

523 without a space between the metavar and the elipsis, but we prefer a 

524 space between. 

525 

526 Does not get called for some option types (e.g. flag) so metavar 

527 transformation that must apply to all types should be applied in 

528 get_help_record. 

529 """ 

530 metavar = super().make_metavar() 

531 if self.multiple and self.nargs == 1: 

532 metavar += " ..." 

533 elif self.nargs != 1: 

534 metavar = f"{metavar[:-3]} ..." 

535 return metavar 

536 

537 

538class MWArgument(click.Argument): 

539 """Overrides click.Argument with desired behaviors.""" 

540 

541 def make_metavar(self): 

542 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

543 help menu. Always adds a space and an elipsis (' ...') after the 

544 metavar name if the option accepts multiple inputs. 

545 

546 By default click adds an elipsis without a space between the metavar 

547 and the elipsis, but we prefer a space between. 

548 

549 Returns 

550 ------- 

551 metavar : `str` 

552 The metavar value. 

553 """ 

554 metavar = super().make_metavar() 

555 if self.nargs != 1: 

556 metavar = f"{metavar[:-3]} ..." 

557 return metavar 

558 

559 

560class OptionSection(MWOption): 

561 """Implements an Option that prints a section label in the help text and 

562 does not pass any value to the command function. 

563 

564 This class does a bit of hackery to add a section label to a click command 

565 help output: first, `expose_value` is set to `False` so that no value is 

566 passed to the command function. Second, this class overrides 

567 `click.Option.get_help_record` to return the section label string without 

568 any prefix so that it stands out as a section label. 

569 

570 The intention for this implementation is to do minimally invasive overrides 

571 of the click classes so as to be robust and easy to fix if the click 

572 internals change. 

573 

574 Parameters 

575 ---------- 

576 sectionName : `str` 

577 The parameter declaration for this option. It is not shown to the user, 

578 it must be unique within the command. If using the `section` decorator 

579 to add a section to a command's options, the section name is 

580 auto-generated. 

581 sectionText : `str` 

582 The text to print in the section identifier. 

583 """ 

584 

585 def __init__(self, sectionName, sectionText): 

586 super().__init__(sectionName, expose_value=False) 

587 self.sectionText = sectionText 

588 

589 def get_help_record(self, ctx): 

590 return (self.sectionText, "") 

591 

592 

593class MWOptionDecorator: 

594 """Wraps the click.option decorator to enable shared options to be declared 

595 and allows inspection of the shared option. 

596 """ 

597 

598 def __init__(self, *param_decls, **kwargs): 

599 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), 

600 **kwargs) 

601 opt = click.Option(param_decls, **kwargs) 

602 self._name = opt.name 

603 self._opts = opt.opts 

604 

605 def name(self): 

606 """Get the name that will be passed to the command function for this 

607 option.""" 

608 return self._name 

609 

610 def opts(self): 

611 """Get the flags that will be used for this option on the command 

612 line.""" 

613 return self._opts 

614 

615 def __call__(self, *args, **kwargs): 

616 return self.partialOpt(*args, **kwargs) 

617 

618 

619class MWArgumentDecorator: 

620 """Wraps the click.argument decorator to enable shared arguments to be 

621 declared. """ 

622 

623 def __init__(self, *param_decls, **kwargs): 

624 self._helpText = kwargs.pop("help", None) 

625 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

626 

627 def __call__(self, *args, help=None, **kwargs): 

628 def decorator(f): 

629 if help is not None: 

630 self._helpText = help 

631 if self._helpText: 631 ↛ 633line 631 didn't jump to line 633, because the condition on line 631 was never false

632 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

633 return self.partialArg(*args, **kwargs)(f) 

634 return decorator 

635 

636 

637class MWCommand(click.Command): 

638 """Command subclass that stores a copy of the args list for use by the 

639 command.""" 

640 

641 def parse_args(self, ctx, args): 

642 MWCtxObj.getFrom(ctx).args = copy.copy(args) 

643 super().parse_args(ctx, args) 

644 

645 

646class MWCtxObj(): 

647 """Helper object for managing the `click.Context.obj` parameter, allows 

648 obj data to be managed in a consistent way. 

649 

650 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

651 initialize the obj if needed and return a new or existing MWCtxObj. 

652 

653 Attributes 

654 ---------- 

655 args : `list` [`str`] 

656 The list of arguments (argument values, option flags, and option 

657 values), split using whitespace, that were passed in on the command 

658 line for the subcommand represented by the parent context object. 

659 """ 

660 

661 def __init__(self): 

662 

663 self.args = None 

664 

665 @staticmethod 

666 def getFrom(ctx): 

667 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the 

668 new or already existing MWCtxObj.""" 

669 if ctx.obj is not None: 

670 return ctx.obj 

671 ctx.obj = MWCtxObj() 

672 return ctx.obj 

673 

674 

675def yaml_presets(ctx, param, value): 

676 """Click callback that reads additional values from the supplied 

677 YAML file. 

678 

679 Parameters 

680 ---------- 

681 ctx : `click.context` 

682 The context for the click operation. Used to extract the subcommand 

683 name. 

684 param : `str` 

685 The parameter name. 

686 value : `object` 

687 The value of the parameter. 

688 """ 

689 ctx.default_map = ctx.default_map or {} 

690 cmd_name = ctx.info_name 

691 if value: 

692 try: 

693 overrides = _read_yaml_presets(value, cmd_name) 

694 except Exception as e: 

695 raise click.BadOptionUsage(param.name, f"Error reading overrides file: {e}", ctx) 

696 # Override the defaults for this subcommand 

697 ctx.default_map.update(overrides) 

698 return 

699 

700 

701def _read_yaml_presets(file_uri, cmd_name): 

702 """Read file command line overrides from YAML config file. 

703 

704 Parameters 

705 ---------- 

706 file_uri : `str` 

707 URI of override YAML file containing the command line overrides. 

708 They should be grouped by command name. 

709 cmd_name : `str` 

710 The subcommand name that is being modified. 

711 

712 Returns 

713 ------- 

714 overrides : `dict` of [`str`, Any] 

715 The relevant command line options read from the override file. 

716 """ 

717 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

718 config = Config(file_uri) 

719 return config[cmd_name]