Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22import click 

23import click.testing 

24from contextlib import contextmanager 

25import copy 

26from functools import partial 

27import io 

28import os 

29import textwrap 

30import traceback 

31from unittest.mock import MagicMock, patch 

32import uuid 

33import yaml 

34 

35from .cliLog import CliLog 

36from ..core.utils import iterable 

37 

38 

39# CLI_MOCK_ENV is set by some tests as an environment variable, it 

40# indicates to the cli_handle_exception function that instead of executing the 

41# command implementation function it should use the Mocker class for unit test 

42# verification. 

43mockEnvVarKey = "CLI_MOCK_ENV" 

44mockEnvVar = {mockEnvVarKey: "1"} 

45 

46# This is used as the metavar argument to Options that accept multiple string 

47# inputs, which may be comma-separarated. For example: 

48# --my-opt foo,bar --my-opt baz. 

49# Other arguments to the Option should include multiple=true and 

50# callback=split_kv. 

51typeStrAcceptsMultiple = "TEXT ..." 

52typeStrAcceptsSingle = "TEXT" 

53 

54 

55def textTypeStr(multiple): 

56 """Get the text type string for CLI help documentation. 

57 

58 Parameters 

59 ---------- 

60 multiple : `bool` 

61 True if multiple text values are allowed, False if only one value is 

62 allowed. 

63 

64 Returns 

65 ------- 

66 textTypeStr : `str` 

67 The type string to use. 

68 """ 

69 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

70 

71 

72# For parameters that support key-value inputs, this defines the separator 

73# for those inputs. 

74split_kv_separator = "=" 

75 

76 

77class Mocker: 

78 

79 mock = MagicMock() 

80 

81 def __init__(self, *args, **kwargs): 

82 """Mocker is a helper class for unit tests. It can be imported and 

83 called and later imported again and call can be verified. 

84 

85 For convenience, constructor arguments are forwarded to the call 

86 function. 

87 """ 

88 self.__call__(*args, **kwargs) 

89 

90 def __call__(self, *args, **kwargs): 

91 """Creates a MagicMock and stores it in a static variable that can 

92 later be verified. 

93 """ 

94 Mocker.mock(*args, **kwargs) 

95 

96 @classmethod 

97 def reset(cls): 

98 cls.mock.reset_mock() 

99 

100 

101class LogCliRunner(click.testing.CliRunner): 

102 """A test runner to use when the logging system will be initialized by code 

103 under test, calls CliLog.resetLog(), which undoes any logging setup that 

104 was done with the CliLog interface. 

105 

106 lsst.log modules can not be set back to an uninitialized state (python 

107 logging modules can be set back to NOTSET), instead they are set to 

108 `CliLog.defaultLsstLogLevel`.""" 

109 

110 def invoke(self, *args, **kwargs): 

111 result = super().invoke(*args, **kwargs) 

112 CliLog.resetLog() 

113 return result 

114 

115 

116def clickResultMsg(result): 

117 """Get a standard assert message from a click result 

118 

119 Parameters 

120 ---------- 

121 result : click.Result 

122 The result object returned from click.testing.CliRunner.invoke 

123 

124 Returns 

125 ------- 

126 msg : `str` 

127 The message string. 

128 """ 

129 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

130 if result.exception: 

131 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

132 return msg 

133 

134 

135@contextmanager 

136def command_test_env(runner, commandModule, commandName): 

137 """A context manager that creates (and then cleans up) an environment that 

138 provides a CLI plugin command with the given name. 

139 

140 Parameters 

141 ---------- 

142 runner : click.testing.CliRunner 

143 The test runner to use to create the isolated filesystem. 

144 commandModule : `str` 

145 The importable module that the command can be imported from. 

146 commandName : `str` 

147 The name of the command being published to import. 

148 """ 

149 with runner.isolated_filesystem(): 

150 with open("resources.yaml", "w") as f: 

151 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

152 # Add a colon to the end of the path on the next line, this tests the 

153 # case where the lookup in LoaderCLI._getPluginList generates an empty 

154 # string in one of the list entries and verifies that the empty string 

155 # is properly stripped out. 

156 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

157 yield 

158 

159 

160def addArgumentHelp(doc, helpText): 

161 """Add a Click argument's help message to a function's documentation. 

162 

163 This is needed because click presents arguments in the order the argument 

164 decorators are applied to a function, top down. But, the evaluation of the 

165 decorators happens bottom up, so if arguments just append their help to the 

166 function's docstring, the argument descriptions appear in reverse order 

167 from the order they are applied in. 

168 

169 Parameters 

170 ---------- 

171 doc : `str` 

172 The function's docstring. 

173 helpText : `str` 

174 The argument's help string to be inserted into the function's 

175 docstring. 

176 

177 Returns 

178 ------- 

179 doc : `str` 

180 Updated function documentation. 

181 """ 

182 if doc is None: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true

183 doc = helpText 

184 else: 

185 doclines = doc.splitlines() 

186 doclines.insert(1, helpText) 

187 doclines.insert(1, "\n") 

188 doc = "\n".join(doclines) 

189 return doc 

190 

191 

192def split_commas(context, param, values): 

193 """Process a tuple of values, where each value may contain comma-separated 

194 values, and return a single list of all the passed-in values. 

195 

196 This function can be passed to the 'callback' argument of a click.option to 

197 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). 

198 

199 Parameters 

200 ---------- 

201 context : `click.Context` or `None` 

202 The current execution context. Unused, but Click always passes it to 

203 callbacks. 

204 param : `click.core.Option` or `None` 

205 The parameter being handled. Unused, but Click always passes it to 

206 callbacks. 

207 values : [`str`] 

208 All the values passed for this option. Strings may contain commas, 

209 which will be treated as delimiters for separate values. 

210 

211 Returns 

212 ------- 

213 list of string 

214 The passed in values separated by commas and combined into a single 

215 list. 

216 """ 

217 if values is None: 

218 return values 

219 valueList = [] 

220 for value in iterable(values): 

221 valueList.extend(value.split(",")) 

222 return tuple(valueList) 

223 

224 

225def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=", 

226 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False): 

227 """Process a tuple of values that are key-value pairs separated by a given 

228 separator. Multiple pairs may be comma separated. Return a dictionary of 

229 all the passed-in values. 

230 

231 This function can be passed to the 'callback' argument of a click.option to 

232 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

233 

234 Parameters 

235 ---------- 

236 context : `click.Context` or `None` 

237 The current execution context. Unused, but Click always passes it to 

238 callbacks. 

239 param : `click.core.Option` or `None` 

240 The parameter being handled. Unused, but Click always passes it to 

241 callbacks. 

242 values : [`str`] 

243 All the values passed for this option. Strings may contain commas, 

244 which will be treated as delimiters for separate values. 

245 choice : `click.Choice`, optional 

246 If provided, verify each value is a valid choice using the provided 

247 `click.Choice` instance. If None, no verification will be done. By 

248 default None 

249 multiple : `bool`, optional 

250 If true, the value may contain multiple comma-separated values. By 

251 default True. 

252 normalize : `bool`, optional 

253 If True and `choice.case_sensitive == False`, normalize the string the 

254 user provided to match the choice's case. By default False. 

255 separator : str, optional 

256 The character that separates key-value pairs. May not be a comma or an 

257 empty space (for space separators use Click's default implementation 

258 for tuples; `type=(str, str)`). By default "=". 

259 unseparated_okay : `bool`, optional 

260 If True, allow values that do not have a separator. They will be 

261 returned in the values dict as a tuple of values in the key '', that 

262 is: `values[''] = (unseparated_values, )`. By default False. 

263 return_type : `type`, must be `dict` or `tuple` 

264 The type of the value that should be returned. 

265 If `dict` then the returned object will be a dict, for each item in 

266 values, the value to the left of the separator will be the key and the 

267 value to the right of the separator will be the value. 

268 If `tuple` then the returned object will be a tuple. Each item in the 

269 tuple will be 2-item tuple, the first item will be the value to the 

270 left of the separator and the second item will be the value to the 

271 right. By default `dict`. 

272 default_key : `Any` 

273 The key to use if a value is passed that is not a key-value pair. 

274 (Passing values that are not key-value pairs requires 

275 ``unseparated_okay`` to be `True`.) 

276 reverse_kv : bool 

277 If true then for each item in values, the value to the left of the 

278 separator is treated as the value and the value to the right of the 

279 separator is treated as the key. By default False. 

280 

281 Returns 

282 ------- 

283 values : `dict` [`str`, `str`] 

284 The passed-in values in dict form. 

285 

286 Raises 

287 ------ 

288 `click.ClickException` 

289 Raised if the separator is not found in an entry, or if duplicate keys 

290 are encountered. 

291 """ 

292 

293 def norm(val): 

294 """If `normalize` is True and `choice` is not `None`, find the value 

295 in the available choices and return the value as spelled in the 

296 choices. 

297 

298 Assumes that val exists in choices; `split_kv` uses the `choice` 

299 instance to verify val is a valid choice. 

300 """ 

301 if normalize and choice is not None: 

302 v = val.casefold() 

303 for opt in choice.choices: 

304 if opt.casefold() == v: 

305 return opt 

306 return val 

307 

308 class RetDict: 

309 

310 def __init__(self): 

311 self.ret = {} 

312 

313 def add(self, key, val): 

314 if reverse_kv: 

315 key, val = val, key 

316 if key in self.ret: 

317 raise click.ClickException(f"Duplicate entries for '{k}' in '{values}'") 

318 self.ret[key] = val 

319 

320 def get(self): 

321 return self.ret 

322 

323 class RetTuple: 

324 

325 def __init__(self): 

326 self.ret = [] 

327 

328 def add(self, key, val): 

329 if reverse_kv: 

330 key, val = val, key 

331 self.ret.append((key, val)) 

332 

333 def get(self): 

334 return tuple(self.ret) 

335 

336 if separator in (",", " "): 

337 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

338 vals = values # preserve the original argument for error reporting below. 

339 if return_type is dict: 

340 ret = RetDict() 

341 elif return_type is tuple: 

342 ret = RetTuple() 

343 else: 

344 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.") 

345 if multiple: 

346 vals = split_commas(context, param, vals) 

347 for val in iterable(vals): 

348 if unseparated_okay and separator not in val: 

349 if choice is not None: 

350 choice(val) # will raise if val is an invalid choice 

351 ret.add(default_key, norm(val)) 

352 else: 

353 try: 

354 k, v = val.split(separator) 

355 if choice is not None: 

356 choice(v) # will raise if val is an invalid choice 

357 except ValueError: 

358 raise click.ClickException( 

359 f"Could not parse key-value pair '{val}' using separator '{separator}', " 

360 f"with multiple values {'allowed' if multiple else 'not allowed'}.") 

361 ret.add(k, norm(v)) 

362 return ret.get() 

363 

364 

365def to_upper(context, param, value): 

366 """Convert a value to upper case. 

367 

368 Parameters 

369 ---------- 

370 context : click.Context 

371 

372 values : string 

373 The value to be converted. 

374 

375 Returns 

376 ------- 

377 string 

378 A copy of the passed-in value, converted to upper case. 

379 """ 

380 return value.upper() 

381 

382 

383def unwrap(val): 

384 """Remove newlines and leading whitespace from a multi-line string with 

385 a consistent indentation level. 

386 

387 The first line of the string may be only a newline or may contain text 

388 followed by a newline, either is ok. After the first line, each line must 

389 begin with a consistant amount of whitespace. So, content of a 

390 triple-quoted string may begin immediately after the quotes, or the string 

391 may start with a newline. Each line after that must be the same amount of 

392 indentation/whitespace followed by text and a newline. The last line may 

393 end with a new line but is not required to do so. 

394 

395 Parameters 

396 ---------- 

397 val : `str` 

398 The string to change. 

399 

400 Returns 

401 ------- 

402 strippedString : `str` 

403 The string with newlines, indentation, and leading and trailing 

404 whitespace removed. 

405 """ 

406 def splitSection(val): 

407 if not val.startswith("\n"): 407 ↛ 411line 407 didn't jump to line 411, because the condition on line 407 was never false

408 firstLine, _, val = val.partition("\n") 

409 firstLine += " " 

410 else: 

411 firstLine = "" 

412 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

413 

414 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

415 

416 

417def cli_handle_exception(func, *args, **kwargs): 

418 """Wrap a function call in an exception handler that raises a 

419 ClickException if there is an Exception. 

420 

421 Also provides support for unit testing by testing for an environment 

422 variable, and if it is present prints the function name, args, and kwargs 

423 to stdout so they can be read and verified by the unit test code. 

424 

425 Parameters 

426 ---------- 

427 func : function 

428 A function to be called and exceptions handled. Will pass args & kwargs 

429 to the function. 

430 

431 Returns 

432 ------- 

433 The result of calling func. 

434 

435 Raises 

436 ------ 

437 click.ClickException 

438 An exception to be handled by the Click CLI tool. 

439 """ 

440 if mockEnvVarKey in os.environ: 

441 Mocker(*args, **kwargs) 

442 return 

443 try: 

444 return func(*args, **kwargs) 

445 except Exception: 

446 msg = io.StringIO() 

447 msg.write("An error occurred during command execution:\n") 

448 traceback.print_exc(file=msg) 

449 raise click.ClickException(msg.getvalue()) 

450 

451 

452class option_section: # noqa: N801 

453 """Decorator to add a section label between options in the help text of a 

454 command. 

455 

456 Parameters 

457 ---------- 

458 sectionText : `str` 

459 The text to print in the section identifier. 

460 """ 

461 

462 def __init__(self, sectionText): 

463 self.sectionText = "\n" + sectionText 

464 

465 def __call__(self, f): 

466 # Generate a parameter declaration that will be unique for this 

467 # section. 

468 return click.option(f"--option-section-{str(uuid.uuid4())}", 

469 sectionText=self.sectionText, 

470 cls=OptionSection)(f) 

471 

472 

473class MWPath(click.Path): 

474 """Overrides click.Path to implement file-does-not-exist checking. 

475 

476 Changes the definition of ``exists` so that `True` indicates the location 

477 (file or directory) must exist, `False` indicates the location must *not* 

478 exist, and `None` indicates that the file may exist or not. The standard 

479 definition for the `click.Path` ``exists`` parameter is that for `True` a 

480 location must exist, but `False` means it is not required to exist (not 

481 that it is required to not exist). 

482 

483 Parameters 

484 ---------- 

485 exists : `True`, `False`, or `None` 

486 If `True`, the location (file or directory) indicated by the caller 

487 must exist. If `False` the location must not exist. If `None`, the 

488 location may exist or not. 

489 

490 For other parameters see `click.Path`. 

491 """ 

492 

493 def __init__(self, exists=None, file_okay=True, dir_okay=True, 

494 writable=False, readable=True, resolve_path=False, 

495 allow_dash=False, path_type=None): 

496 self.mustNotExist = exists is False 

497 if exists is None: 

498 exists = False 

499 super().__init__(exists, file_okay, dir_okay, writable, readable, 

500 resolve_path, allow_dash, path_type) 

501 

502 def convert(self, value, param, ctx): 

503 """Called by click.ParamType to "convert values through types". 

504 `click.Path` uses this step to verify Path conditions.""" 

505 if self.mustNotExist and os.path.exists(value): 

506 self.fail(f'{self.path_type} "{value}" should not exist.') 

507 return super().convert(value, param, ctx) 

508 

509 

510class MWOption(click.Option): 

511 """Overrides click.Option with desired behaviors.""" 

512 

513 def make_metavar(self): 

514 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

515 help menu. Adds a space and an elipsis after the metavar name if 

516 the option accepts multiple inputs, otherwise defers to the base 

517 implementation. 

518 

519 By default click does not add an elipsis when multiple is True and 

520 nargs is 1. And when nargs does not equal 1 click adds an elipsis 

521 without a space between the metavar and the elipsis, but we prefer a 

522 space between. 

523 

524 Does not get called for some option types (e.g. flag) so metavar 

525 transformation that must apply to all types should be applied in 

526 get_help_record. 

527 """ 

528 metavar = super().make_metavar() 

529 if self.multiple and self.nargs == 1: 

530 metavar += " ..." 

531 elif self.nargs != 1: 

532 metavar = f"{metavar[:-3]} ..." 

533 return metavar 

534 

535 

536class MWArgument(click.Argument): 

537 """Overrides click.Argument with desired behaviors.""" 

538 

539 def make_metavar(self): 

540 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

541 help menu. Always adds a space and an elipsis (' ...') after the 

542 metavar name if the option accepts multiple inputs. 

543 

544 By default click adds an elipsis without a space between the metavar 

545 and the elipsis, but we prefer a space between. 

546 

547 Returns 

548 ------- 

549 metavar : `str` 

550 The metavar value. 

551 """ 

552 metavar = super().make_metavar() 

553 if self.nargs != 1: 

554 metavar = f"{metavar[:-3]} ..." 

555 return metavar 

556 

557 

558class OptionSection(MWOption): 

559 """Implements an Option that prints a section label in the help text and 

560 does not pass any value to the command function. 

561 

562 This class does a bit of hackery to add a section label to a click command 

563 help output: first, `expose_value` is set to `False` so that no value is 

564 passed to the command function. Second, this class overrides 

565 `click.Option.get_help_record` to return the section label string without 

566 any prefix so that it stands out as a section label. 

567 

568 The intention for this implementation is to do minimally invasive overrides 

569 of the click classes so as to be robust and easy to fix if the click 

570 internals change. 

571 

572 Parameters 

573 ---------- 

574 sectionName : `str` 

575 The parameter declaration for this option. It is not shown to the user, 

576 it must be unique within the command. If using the `section` decorator 

577 to add a section to a command's options, the section name is 

578 auto-generated. 

579 sectionText : `str` 

580 The text to print in the section identifier. 

581 """ 

582 

583 def __init__(self, sectionName, sectionText): 

584 super().__init__(sectionName, expose_value=False) 

585 self.sectionText = sectionText 

586 

587 def get_help_record(self, ctx): 

588 return (self.sectionText, "") 

589 

590 

591class MWOptionDecorator: 

592 """Wraps the click.option decorator to enable shared options to be declared 

593 and allows inspection of the shared option. 

594 """ 

595 

596 def __init__(self, *param_decls, **kwargs): 

597 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), 

598 **kwargs) 

599 opt = click.Option(param_decls, **kwargs) 

600 self._name = opt.name 

601 self._opts = opt.opts 

602 

603 def name(self): 

604 """Get the name that will be passed to the command function for this 

605 option.""" 

606 return self._name 

607 

608 def opts(self): 

609 """Get the flags that will be used for this option on the command 

610 line.""" 

611 return self._opts 

612 

613 def __call__(self, *args, **kwargs): 

614 return self.partialOpt(*args, **kwargs) 

615 

616 

617class MWArgumentDecorator: 

618 """Wraps the click.argument decorator to enable shared arguments to be 

619 declared. """ 

620 

621 def __init__(self, *param_decls, **kwargs): 

622 self._helpText = kwargs.pop("help", None) 

623 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

624 

625 def __call__(self, *args, help=None, **kwargs): 

626 def decorator(f): 

627 if help is not None: 

628 self._helpText = help 

629 if self._helpText: 629 ↛ 631line 629 didn't jump to line 631, because the condition on line 629 was never false

630 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

631 return self.partialArg(*args, **kwargs)(f) 

632 return decorator 

633 

634 

635class MWCommand(click.Command): 

636 """Command subclass that stores a copy of the args list for use by the 

637 command.""" 

638 

639 def parse_args(self, ctx, args): 

640 MWCtxObj.getFrom(ctx).args = copy.copy(args) 

641 super().parse_args(ctx, args) 

642 

643 

644class MWCtxObj(): 

645 """Helper object for managing the `click.Context.obj` parameter, allows 

646 obj data to be managed in a consistent way. 

647 

648 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

649 initialize the obj if needed and return a new or existing MWCtxObj. 

650 

651 Attributes 

652 ---------- 

653 args : `list` [`str`] 

654 The list of arguments (argument values, option flags, and option 

655 values), split using whitespace, that were passed in on the command 

656 line for the subcommand represented by the parent context object. 

657 """ 

658 

659 def __init__(self): 

660 

661 self.args = None 

662 

663 @staticmethod 

664 def getFrom(ctx): 

665 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the 

666 new or already existing MWCtxObj.""" 

667 if ctx.obj is not None: 

668 return ctx.obj 

669 ctx.obj = MWCtxObj() 

670 return ctx.obj