Coverage for python/lsst/daf/butler/cli/utils.py: 38%

317 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-08-03 02:30 -0700

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23__all__ = ( 

24 "astropyTablesToStr", 

25 "printAstropyTables", 

26 "textTypeStr", 

27 "LogCliRunner", 

28 "clickResultMsg", 

29 "command_test_env", 

30 "addArgumentHelp", 

31 "split_commas", 

32 "split_kv", 

33 "to_upper", 

34 "unwrap", 

35 "option_section", 

36 "MWPath", 

37 "MWOption", 

38 "MWArgument", 

39 "OptionSection", 

40 "MWOptionDecorator", 

41 "MWArgumentDecorator", 

42 "MWCommand", 

43 "ButlerCommand", 

44 "OptionGroup", 

45 "MWCtxObj", 

46 "yaml_presets", 

47 "sortAstropyTable", 

48 "catch_and_exit", 

49) 

50 

51 

52import itertools 

53import logging 

54import os 

55import sys 

56import textwrap 

57import traceback 

58import uuid 

59from collections import Counter 

60from contextlib import contextmanager 

61from functools import partial, wraps 

62from unittest.mock import patch 

63 

64import click 

65import click.exceptions 

66import click.testing 

67import yaml 

68from lsst.utils.iteration import ensure_iterable 

69 

70from ..core.config import Config 

71from .cliLog import CliLog 

72 

73log = logging.getLogger(__name__) 

74 

75# This is used as the metavar argument to Options that accept multiple string 

76# inputs, which may be comma-separarated. For example: 

77# --my-opt foo,bar --my-opt baz. 

78# Other arguments to the Option should include multiple=true and 

79# callback=split_kv. 

80typeStrAcceptsMultiple = "TEXT ..." 

81typeStrAcceptsSingle = "TEXT" 

82 

83# The standard help string for the --where option when it takes a WHERE clause. 

84where_help = ( 

85 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

86 "dimension table or a dimension name as a shortcut for the primary key column of a " 

87 "dimension table." 

88) 

89 

90 

91def astropyTablesToStr(tables): 

92 """Render astropy tables to string as they are displayed in the CLI. 

93 

94 Output formatting matches ``printAstropyTables``. 

95 """ 

96 ret = "" 

97 for table in tables: 

98 ret += "\n" 

99 table.pformat_all() 

100 ret += "\n" 

101 return ret 

102 

103 

104def printAstropyTables(tables): 

105 """Print astropy tables to be displayed in the CLI. 

106 

107 Output formatting matches ``astropyTablesToStr``. 

108 """ 

109 for table in tables: 

110 print("") 

111 table.pprint_all() 

112 print("") 

113 

114 

115def textTypeStr(multiple): 

116 """Get the text type string for CLI help documentation. 

117 

118 Parameters 

119 ---------- 

120 multiple : `bool` 

121 True if multiple text values are allowed, False if only one value is 

122 allowed. 

123 

124 Returns 

125 ------- 

126 textTypeStr : `str` 

127 The type string to use. 

128 """ 

129 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

130 

131 

132class LogCliRunner(click.testing.CliRunner): 

133 """A test runner to use when the logging system will be initialized by code 

134 under test, calls CliLog.resetLog(), which undoes any logging setup that 

135 was done with the CliLog interface. 

136 

137 lsst.log modules can not be set back to an uninitialized state (python 

138 logging modules can be set back to NOTSET), instead they are set to 

139 `CliLog.defaultLsstLogLevel`.""" 

140 

141 def invoke(self, *args, **kwargs): 

142 result = super().invoke(*args, **kwargs) 

143 CliLog.resetLog() 

144 return result 

145 

146 

147def clickResultMsg(result): 

148 """Get a standard assert message from a click result 

149 

150 Parameters 

151 ---------- 

152 result : click.Result 

153 The result object returned from click.testing.CliRunner.invoke 

154 

155 Returns 

156 ------- 

157 msg : `str` 

158 The message string. 

159 """ 

160 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

161 if result.exception: 

162 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

163 return msg 

164 

165 

166@contextmanager 

167def command_test_env(runner, commandModule, commandName): 

168 """A context manager that creates (and then cleans up) an environment that 

169 provides a CLI plugin command with the given name. 

170 

171 Parameters 

172 ---------- 

173 runner : click.testing.CliRunner 

174 The test runner to use to create the isolated filesystem. 

175 commandModule : `str` 

176 The importable module that the command can be imported from. 

177 commandName : `str` 

178 The name of the command being published to import. 

179 """ 

180 with runner.isolated_filesystem(): 

181 with open("resources.yaml", "w") as f: 

182 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

183 # Add a colon to the end of the path on the next line, this tests the 

184 # case where the lookup in LoaderCLI._getPluginList generates an empty 

185 # string in one of the list entries and verifies that the empty string 

186 # is properly stripped out. 

187 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

188 yield 

189 

190 

191def addArgumentHelp(doc, helpText): 

192 """Add a Click argument's help message to a function's documentation. 

193 

194 This is needed because click presents arguments in the order the argument 

195 decorators are applied to a function, top down. But, the evaluation of the 

196 decorators happens bottom up, so if arguments just append their help to the 

197 function's docstring, the argument descriptions appear in reverse order 

198 from the order they are applied in. 

199 

200 Parameters 

201 ---------- 

202 doc : `str` 

203 The function's docstring. 

204 helpText : `str` 

205 The argument's help string to be inserted into the function's 

206 docstring. 

207 

208 Returns 

209 ------- 

210 doc : `str` 

211 Updated function documentation. 

212 """ 

213 if doc is None: 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true

214 doc = helpText 

215 else: 

216 # See click documentation for details: 

217 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

218 # In short, text for the click command help can be truncated by putting 

219 # "\f" in the docstring, everything after it should be removed 

220 if "\f" in doc: 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true

221 doc = doc.split("\f")[0] 

222 

223 doclines = doc.splitlines() 

224 # The function's docstring may span multiple lines, so combine the 

225 # docstring from all the first lines until a blank line is encountered. 

226 # (Lines after the first blank line will be argument help.) 

227 while len(doclines) > 1 and doclines[1]: 

228 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

229 # Add standard indent to help text for proper alignment with command 

230 # function documentation: 

231 helpText = " " + helpText 

232 doclines.insert(1, helpText) 

233 doclines.insert(1, "\n") 

234 doc = "\n".join(doclines) 

235 return doc 

236 

237 

238def split_commas(context, param, values): 

239 """Process a tuple of values, where each value may contain comma-separated 

240 values, and return a single list of all the passed-in values. 

241 

242 This function can be passed to the 'callback' argument of a click.option to 

243 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). 

244 

245 Parameters 

246 ---------- 

247 context : `click.Context` or `None` 

248 The current execution context. Unused, but Click always passes it to 

249 callbacks. 

250 param : `click.core.Option` or `None` 

251 The parameter being handled. Unused, but Click always passes it to 

252 callbacks. 

253 values : [`str`] 

254 All the values passed for this option. Strings may contain commas, 

255 which will be treated as delimiters for separate values. 

256 

257 Returns 

258 ------- 

259 list of string 

260 The passed in values separated by commas and combined into a single 

261 list. 

262 """ 

263 if values is None: 

264 return values 

265 valueList = [] 

266 for value in ensure_iterable(values): 

267 valueList.extend(value.split(",")) 

268 return tuple(valueList) 

269 

270 

271def split_kv( 

272 context, 

273 param, 

274 values, 

275 choice=None, 

276 multiple=True, 

277 normalize=False, 

278 separator="=", 

279 unseparated_okay=False, 

280 return_type=dict, 

281 default_key="", 

282 reverse_kv=False, 

283 add_to_default=False, 

284): 

285 """Process a tuple of values that are key-value pairs separated by a given 

286 separator. Multiple pairs may be comma separated. Return a dictionary of 

287 all the passed-in values. 

288 

289 This function can be passed to the 'callback' argument of a click.option to 

290 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

291 

292 Parameters 

293 ---------- 

294 context : `click.Context` or `None` 

295 The current execution context. Unused, but Click always passes it to 

296 callbacks. 

297 param : `click.core.Option` or `None` 

298 The parameter being handled. Unused, but Click always passes it to 

299 callbacks. 

300 values : [`str`] 

301 All the values passed for this option. Strings may contain commas, 

302 which will be treated as delimiters for separate values. 

303 choice : `click.Choice`, optional 

304 If provided, verify each value is a valid choice using the provided 

305 `click.Choice` instance. If None, no verification will be done. By 

306 default None 

307 multiple : `bool`, optional 

308 If true, the value may contain multiple comma-separated values. By 

309 default True. 

310 normalize : `bool`, optional 

311 If True and `choice.case_sensitive == False`, normalize the string the 

312 user provided to match the choice's case. By default False. 

313 separator : str, optional 

314 The character that separates key-value pairs. May not be a comma or an 

315 empty space (for space separators use Click's default implementation 

316 for tuples; `type=(str, str)`). By default "=". 

317 unseparated_okay : `bool`, optional 

318 If True, allow values that do not have a separator. They will be 

319 returned in the values dict as a tuple of values in the key '', that 

320 is: `values[''] = (unseparated_values, )`. By default False. 

321 return_type : `type`, must be `dict` or `tuple` 

322 The type of the value that should be returned. 

323 If `dict` then the returned object will be a dict, for each item in 

324 values, the value to the left of the separator will be the key and the 

325 value to the right of the separator will be the value. 

326 If `tuple` then the returned object will be a tuple. Each item in the 

327 tuple will be 2-item tuple, the first item will be the value to the 

328 left of the separator and the second item will be the value to the 

329 right. By default `dict`. 

330 default_key : `Any` 

331 The key to use if a value is passed that is not a key-value pair. 

332 (Passing values that are not key-value pairs requires 

333 ``unseparated_okay`` to be `True`.) 

334 reverse_kv : bool 

335 If true then for each item in values, the value to the left of the 

336 separator is treated as the value and the value to the right of the 

337 separator is treated as the key. By default False. 

338 add_to_default : `bool`, optional 

339 If True, then passed-in values will not overwrite the default value 

340 unless the ``return_type`` is `dict` and passed-in value(s) have the 

341 same key(s) as the default value. 

342 

343 Returns 

344 ------- 

345 values : `dict` [`str`, `str`] 

346 The passed-in values in dict form. 

347 

348 Raises 

349 ------ 

350 `click.ClickException` 

351 Raised if the separator is not found in an entry, or if duplicate keys 

352 are encountered. 

353 """ 

354 

355 def norm(val): 

356 """If `normalize` is True and `choice` is not `None`, find the value 

357 in the available choices and return the value as spelled in the 

358 choices. 

359 

360 Assumes that val exists in choices; `split_kv` uses the `choice` 

361 instance to verify val is a valid choice. 

362 """ 

363 if normalize and choice is not None: 

364 v = val.casefold() 

365 for opt in choice.choices: 

366 if opt.casefold() == v: 

367 return opt 

368 return val 

369 

370 class RetDict: 

371 def __init__(self): 

372 self.ret = {} 

373 

374 def add(self, key, val): 

375 if reverse_kv: 

376 key, val = val, key 

377 self.ret[key] = val 

378 

379 def get(self): 

380 return self.ret 

381 

382 class RetTuple: 

383 def __init__(self): 

384 self.ret = [] 

385 

386 def add(self, key, val): 

387 if reverse_kv: 

388 key, val = val, key 

389 self.ret.append((key, val)) 

390 

391 def get(self): 

392 return tuple(self.ret) 

393 

394 if separator in (",", " "): 

395 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

396 vals = values # preserve the original argument for error reporting below. 

397 

398 if add_to_default: 

399 default = param.get_default(context) 

400 if default: 

401 vals = itertools.chain(default, vals) 

402 

403 if return_type is dict: 

404 ret = RetDict() 

405 elif return_type is tuple: 

406 ret = RetTuple() 

407 else: 

408 raise click.ClickException( 

409 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

410 ) 

411 if multiple: 

412 vals = split_commas(context, param, vals) 

413 for val in ensure_iterable(vals): 

414 if unseparated_okay and separator not in val: 

415 if choice is not None: 

416 choice(val) # will raise if val is an invalid choice 

417 ret.add(default_key, norm(val)) 

418 else: 

419 try: 

420 k, v = val.split(separator) 

421 if choice is not None: 

422 choice(v) # will raise if val is an invalid choice 

423 except ValueError: 

424 raise click.ClickException( 

425 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

426 f"with multiple values {'allowed' if multiple else 'not allowed'}." 

427 ) 

428 ret.add(k, norm(v)) 

429 return ret.get() 

430 

431 

432def to_upper(context, param, value): 

433 """Convert a value to upper case. 

434 

435 Parameters 

436 ---------- 

437 context : click.Context 

438 

439 values : string 

440 The value to be converted. 

441 

442 Returns 

443 ------- 

444 string 

445 A copy of the passed-in value, converted to upper case. 

446 """ 

447 return value.upper() 

448 

449 

450def unwrap(val): 

451 """Remove newlines and leading whitespace from a multi-line string with 

452 a consistent indentation level. 

453 

454 The first line of the string may be only a newline or may contain text 

455 followed by a newline, either is ok. After the first line, each line must 

456 begin with a consistant amount of whitespace. So, content of a 

457 triple-quoted string may begin immediately after the quotes, or the string 

458 may start with a newline. Each line after that must be the same amount of 

459 indentation/whitespace followed by text and a newline. The last line may 

460 end with a new line but is not required to do so. 

461 

462 Parameters 

463 ---------- 

464 val : `str` 

465 The string to change. 

466 

467 Returns 

468 ------- 

469 strippedString : `str` 

470 The string with newlines, indentation, and leading and trailing 

471 whitespace removed. 

472 """ 

473 

474 def splitSection(val): 

475 if not val.startswith("\n"): 475 ↛ 479line 475 didn't jump to line 479, because the condition on line 475 was never false

476 firstLine, _, val = val.partition("\n") 

477 firstLine += " " 

478 else: 

479 firstLine = "" 

480 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

481 

482 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

483 

484 

485class option_section: # noqa: N801 

486 """Decorator to add a section label between options in the help text of a 

487 command. 

488 

489 Parameters 

490 ---------- 

491 sectionText : `str` 

492 The text to print in the section identifier. 

493 """ 

494 

495 def __init__(self, sectionText): 

496 self.sectionText = "\n" + sectionText 

497 

498 def __call__(self, f): 

499 # Generate a parameter declaration that will be unique for this 

500 # section. 

501 return click.option( 

502 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

503 )(f) 

504 

505 

506class MWPath(click.Path): 

507 """Overrides click.Path to implement file-does-not-exist checking. 

508 

509 Changes the definition of ``exists` so that `True` indicates the location 

510 (file or directory) must exist, `False` indicates the location must *not* 

511 exist, and `None` indicates that the file may exist or not. The standard 

512 definition for the `click.Path` ``exists`` parameter is that for `True` a 

513 location must exist, but `False` means it is not required to exist (not 

514 that it is required to not exist). 

515 

516 Parameters 

517 ---------- 

518 exists : `True`, `False`, or `None` 

519 If `True`, the location (file or directory) indicated by the caller 

520 must exist. If `False` the location must not exist. If `None`, the 

521 location may exist or not. 

522 

523 For other parameters see `click.Path`. 

524 """ 

525 

526 def __init__( 

527 self, 

528 exists=None, 

529 file_okay=True, 

530 dir_okay=True, 

531 writable=False, 

532 readable=True, 

533 resolve_path=False, 

534 allow_dash=False, 

535 path_type=None, 

536 ): 

537 self.mustNotExist = exists is False 

538 if exists is None: 538 ↛ 540line 538 didn't jump to line 540, because the condition on line 538 was never false

539 exists = False 

540 super().__init__( 

541 exists=exists, 

542 file_okay=file_okay, 

543 dir_okay=dir_okay, 

544 writable=writable, 

545 readable=readable, 

546 resolve_path=resolve_path, 

547 allow_dash=allow_dash, 

548 path_type=path_type, 

549 ) 

550 

551 def convert(self, value, param, ctx): 

552 """Called by click.ParamType to "convert values through types". 

553 `click.Path` uses this step to verify Path conditions.""" 

554 if self.mustNotExist and os.path.exists(value): 

555 self.fail(f'Path "{value}" should not exist.') 

556 return super().convert(value, param, ctx) 

557 

558 

559class MWOption(click.Option): 

560 """Overrides click.Option with desired behaviors.""" 

561 

562 def make_metavar(self): 

563 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

564 help menu. Adds a space and an elipsis after the metavar name if 

565 the option accepts multiple inputs, otherwise defers to the base 

566 implementation. 

567 

568 By default click does not add an elipsis when multiple is True and 

569 nargs is 1. And when nargs does not equal 1 click adds an elipsis 

570 without a space between the metavar and the elipsis, but we prefer a 

571 space between. 

572 

573 Does not get called for some option types (e.g. flag) so metavar 

574 transformation that must apply to all types should be applied in 

575 get_help_record. 

576 """ 

577 metavar = super().make_metavar() 

578 if self.multiple and self.nargs == 1: 

579 metavar += " ..." 

580 elif self.nargs != 1: 

581 metavar = f"{metavar[:-3]} ..." 

582 return metavar 

583 

584 

585class MWArgument(click.Argument): 

586 """Overrides click.Argument with desired behaviors.""" 

587 

588 def make_metavar(self): 

589 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

590 help menu. Always adds a space and an elipsis (' ...') after the 

591 metavar name if the option accepts multiple inputs. 

592 

593 By default click adds an elipsis without a space between the metavar 

594 and the elipsis, but we prefer a space between. 

595 

596 Returns 

597 ------- 

598 metavar : `str` 

599 The metavar value. 

600 """ 

601 metavar = super().make_metavar() 

602 if self.nargs != 1: 

603 metavar = f"{metavar[:-3]} ..." 

604 return metavar 

605 

606 

607class OptionSection(MWOption): 

608 """Implements an Option that prints a section label in the help text and 

609 does not pass any value to the command function. 

610 

611 This class does a bit of hackery to add a section label to a click command 

612 help output: first, `expose_value` is set to `False` so that no value is 

613 passed to the command function. Second, this class overrides 

614 `click.Option.get_help_record` to return the section label string without 

615 any prefix so that it stands out as a section label. 

616 

617 This class overrides the hidden attribute because our documentation build 

618 tool, sphinx-click, implements its own `get_help_record` function which 

619 builds the record from other option values (e.g. `name`, `opts`), which 

620 breaks the hack we use to make `get_help_record` only return the 

621 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

622 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

623 entering its `_get_help_record` function. So, making the hidden property 

624 return True hides this option from sphinx-click, while allowing the section 

625 text to be returned by our `get_help_record` method when using Click. 

626 

627 The intention for this implementation is to do minimally invasive overrides 

628 of the click classes so as to be robust and easy to fix if the click 

629 internals change. 

630 

631 Parameters 

632 ---------- 

633 sectionName : `str` 

634 The parameter declaration for this option. It is not shown to the user, 

635 it must be unique within the command. If using the `section` decorator 

636 to add a section to a command's options, the section name is 

637 auto-generated. 

638 sectionText : `str` 

639 The text to print in the section identifier. 

640 """ 

641 

642 @property 

643 def hidden(self): 

644 return True 

645 

646 @hidden.setter 

647 def hidden(self, val): 

648 pass 

649 

650 def __init__(self, sectionName, sectionText): 

651 super().__init__(sectionName, expose_value=False) 

652 self.sectionText = sectionText 

653 

654 def get_help_record(self, ctx): 

655 return (self.sectionText, "") 

656 

657 

658class MWOptionDecorator: 

659 """Wraps the click.option decorator to enable shared options to be declared 

660 and allows inspection of the shared option. 

661 """ 

662 

663 def __init__(self, *param_decls, **kwargs): 

664 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

665 opt = click.Option(param_decls, **kwargs) 

666 self._name = opt.name 

667 self._opts = opt.opts 

668 

669 def name(self): 

670 """Get the name that will be passed to the command function for this 

671 option.""" 

672 return self._name 

673 

674 def opts(self): 

675 """Get the flags that will be used for this option on the command 

676 line.""" 

677 return self._opts 

678 

679 @property 

680 def help(self): 

681 """Get the help text for this option. Returns an empty string if no 

682 help was defined.""" 

683 return self.partialOpt.keywords.get("help", "") 

684 

685 def __call__(self, *args, **kwargs): 

686 return self.partialOpt(*args, **kwargs) 

687 

688 

689class MWArgumentDecorator: 

690 """Wraps the click.argument decorator to enable shared arguments to be 

691 declared.""" 

692 

693 def __init__(self, *param_decls, **kwargs): 

694 self._helpText = kwargs.pop("help", None) 

695 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

696 

697 def __call__(self, *args, help=None, **kwargs): 

698 def decorator(f): 

699 if help is not None: 

700 self._helpText = help 

701 if self._helpText: 

702 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

703 return self.partialArg(*args, **kwargs)(f) 

704 

705 return decorator 

706 

707 

708class MWCommand(click.Command): 

709 """Command subclass that stores a copy of the args list for use by the 

710 command.""" 

711 

712 extra_epilog = None 

713 

714 def __init__(self, *args, **kwargs): 

715 # wrap callback method with catch_and_exit decorator 

716 callback = kwargs.get("callback") 

717 if callback is not None: 717 ↛ 720line 717 didn't jump to line 720, because the condition on line 717 was never false

718 kwargs = kwargs.copy() 

719 kwargs["callback"] = catch_and_exit(callback) 

720 super().__init__(*args, **kwargs) 

721 

722 def _capture_args(self, ctx, args): 

723 """Capture the command line options and arguments. 

724 

725 See details about what is captured and the order in which it is stored 

726 in the documentation of `MWCtxObj`. 

727 

728 Parameters 

729 ---------- 

730 ctx : `click.Context` 

731 The current Context. 

732 args : `list` [`str`] 

733 The list of arguments from the command line, split at spaces but 

734 not at separators (like "="). 

735 """ 

736 parser = self.make_parser(ctx) 

737 opts, _, param_order = parser.parse_args(args=list(args)) 

738 # `param_order` is a list of click.Option and click.Argument, there is 

739 # one item for each time the Option or Argument was used on the 

740 # command line. Options will precede Arguments, within each sublist 

741 # they are in the order they were used on the command line. Note that 

742 # click.Option and click.Argument do not contain the value from the 

743 # command line; values are in `opts`. 

744 # 

745 # `opts` is a dict where the key is the argument name to the 

746 # click.Command function, this name matches the `click.Option.name` or 

747 # `click.Argument.name`. For Options, an item will only be present if 

748 # the Option was used on the command line. For Arguments, an item will 

749 # always be present and if no value was provided on the command line 

750 # the value will be `None`. If the option accepts multiple values, the 

751 # value in `opts` is a tuple, otherwise it is a single item. 

752 next_idx = Counter() 

753 captured_args = [] 

754 for param in param_order: 

755 if isinstance(param, click.Option): 

756 if param.multiple: 

757 val = opts[param.name][next_idx[param.name]] 

758 next_idx[param.name] += 1 

759 else: 

760 val = opts[param.name] 

761 if param.is_flag: 

762 # Bool options store their True flags in opts and their 

763 # False flags in secondary_opts. 

764 if val: 

765 flag = max(param.opts, key=len) 

766 else: 

767 flag = max(param.secondary_opts, key=len) 

768 captured_args.append(flag) 

769 else: 

770 captured_args.append(max(param.opts, key=len)) 

771 captured_args.append(val) 

772 elif isinstance(param, click.Argument): 

773 if (opt := opts[param.name]) is not None: 

774 captured_args.append(opt) 

775 else: 

776 assert False # All parameters should be an Option or an Argument. 

777 MWCtxObj.getFrom(ctx).args = captured_args 

778 

779 def parse_args(self, ctx, args): 

780 """Given a context and a list of arguments this creates the parser and 

781 parses the arguments, then modifies the context as necessary. This is 

782 automatically invoked by make_context(). 

783 

784 This function overrides `click.Command.parse_args`. 

785 

786 The call to `_capture_args` in this override stores the arguments 

787 (option names, option value, and argument values) that were used by the 

788 caller on the command line in the context object. These stored 

789 arugments can be used by the command function, e.g. to process options 

790 in the order they appeared on the command line (pipetask uses this 

791 feature to create pipeline actions in an order from different options). 

792 

793 Parameters 

794 ---------- 

795 ctx : `click.core.Context` 

796 The current Context.ß 

797 args : `list` [`str`] 

798 The list of arguments from the command line, split at spaces but 

799 not at separators (like "="). 

800 """ 

801 self._capture_args(ctx, args) 

802 super().parse_args(ctx, args) 

803 

804 @property 

805 def epilog(self): 

806 """Override the epilog attribute to add extra_epilog (if defined by a 

807 subclass) to the end of any epilog provided by a subcommand. 

808 """ 

809 ret = self._epilog if self._epilog else "" 

810 if self.extra_epilog: 

811 if ret: 

812 ret += "\n\n" 

813 ret += self.extra_epilog 

814 return ret 

815 

816 @epilog.setter 

817 def epilog(self, val): 

818 self._epilog = val 

819 

820 

821class ButlerCommand(MWCommand): 

822 """Command subclass with butler-command specific overrides.""" 

823 

824 extra_epilog = "See 'butler --help' for more options." 

825 

826 

827class OptionGroup: 

828 """Base class for an option group decorator. Requires the option group 

829 subclass to have a property called `decorator`.""" 

830 

831 def __call__(self, f): 

832 for decorator in reversed(self.decorators): 

833 f = decorator(f) 

834 return f 

835 

836 

837class MWCtxObj: 

838 """Helper object for managing the `click.Context.obj` parameter, allows 

839 obj data to be managed in a consistent way. 

840 

841 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

842 initialize the obj if needed and return a new or existing `MWCtxObj`. 

843 

844 The `args` attribute contains a list of options, option values, and 

845 argument values that is similar to the list of arguments and options that 

846 were passed in on the command line, with differences noted below: 

847 

848 * Option namess and option values are first in the list, and argument 

849 values come last. The order of options and option values is preserved 

850 within the options. The order of argument values is preserved. 

851 

852 * The longest option name is used for the option in the `args` list, e.g. 

853 if an option accepts both short and long names "-o / --option" and the 

854 short option name "-o" was used on the command line, the longer name will 

855 be the one that appears in `args`. 

856 

857 * A long option name (which begins with two dashes "--") and its value may 

858 be separated by an equal sign; the name and value are split at the equal 

859 sign and it is removed. In `args`, the option is in one list item, and 

860 the option value (without the equal sign) is in the next list item. e.g. 

861 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

862 `args`. 

863 

864 * A short option name, (which begins with one dash "-") and its value are 

865 split immediately after the short option name, and if there is 

866 whitespace between the short option name and its value it is removed. 

867 Everything after the short option name (excluding whitespace) is included 

868 in the value. If the `Option` has a long name, the long name will be used 

869 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

870 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

871 `["--option", "=foo"]` (because everything after the short option name, 

872 except whitespace, is used for the value (as is standard with unix 

873 command line tools). 

874 

875 Attributes 

876 ---------- 

877 args : `list` [`str`] 

878 A list of options, option values, and arguments simialr to those that 

879 were passed in on the command line. See comments about captured options 

880 & arguments above. 

881 """ 

882 

883 def __init__(self): 

884 

885 self.args = None 

886 

887 @staticmethod 

888 def getFrom(ctx): 

889 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

890 the new or already existing `MWCtxObj`.""" 

891 if ctx.obj is not None: 

892 return ctx.obj 

893 ctx.obj = MWCtxObj() 

894 return ctx.obj 

895 

896 

897def yaml_presets(ctx, param, value): 

898 """Click callback that reads additional values from the supplied 

899 YAML file. 

900 

901 Parameters 

902 ---------- 

903 ctx : `click.context` 

904 The context for the click operation. Used to extract the subcommand 

905 name and translate option & argument names. 

906 param : `str` 

907 The parameter name. 

908 value : `object` 

909 The value of the parameter. 

910 """ 

911 

912 def _name_for_option(ctx: click.Context, option: str) -> str: 

913 """Use a CLI option name to find the name of the argument to the 

914 command function. 

915 

916 Parameters 

917 ---------- 

918 ctx : `click.Context` 

919 The context for the click operation. 

920 option : `str` 

921 The option/argument name from the yaml file. 

922 

923 Returns 

924 ------- 

925 name : str 

926 The name of the argument to use when calling the click.command 

927 function, as it should appear in the `ctx.default_map`. 

928 

929 Raises 

930 ------ 

931 RuntimeError 

932 Raised if the option name from the yaml file does not exist in the 

933 command parameters. This catches misspellings and incorrect useage 

934 in the yaml file. 

935 """ 

936 for param in ctx.command.params: 

937 # Remove leading dashes: they are not used for option names in the 

938 # yaml file. 

939 if option in [opt.lstrip("-") for opt in param.opts]: 

940 return param.name 

941 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

942 

943 ctx.default_map = ctx.default_map or {} 

944 cmd_name = ctx.info_name 

945 if value: 

946 try: 

947 overrides = _read_yaml_presets(value, cmd_name) 

948 options = list(overrides.keys()) 

949 for option in options: 

950 name = _name_for_option(ctx, option) 

951 if name == option: 

952 continue 

953 overrides[name] = overrides.pop(option) 

954 except Exception as e: 

955 raise click.BadOptionUsage( 

956 option_name=param.name, 

957 message=f"Error reading overrides file: {e}", 

958 ctx=ctx, 

959 ) 

960 # Override the defaults for this subcommand 

961 ctx.default_map.update(overrides) 

962 return 

963 

964 

965def _read_yaml_presets(file_uri, cmd_name): 

966 """Read file command line overrides from YAML config file. 

967 

968 Parameters 

969 ---------- 

970 file_uri : `str` 

971 URI of override YAML file containing the command line overrides. 

972 They should be grouped by command name. 

973 cmd_name : `str` 

974 The subcommand name that is being modified. 

975 

976 Returns 

977 ------- 

978 overrides : `dict` of [`str`, Any] 

979 The relevant command line options read from the override file. 

980 """ 

981 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

982 config = Config(file_uri) 

983 return config[cmd_name] 

984 

985 

986def sortAstropyTable(table, dimensions, sort_first=None): 

987 """Sort an astropy table, with prioritization given to columns in this 

988 order: 

989 1. the provided named columns 

990 2. spatial and temporal columns 

991 3. the rest of the columns 

992 

993 The table is sorted in-place, and is also returned for convenience. 

994 

995 Parameters 

996 ---------- 

997 table : `astropy.table.Table` 

998 The table to sort 

999 dimensions : `list` [``Dimension``] 

1000 The dimensions of the dataIds in the table (the dimensions should be 

1001 the same for all the dataIds). Used to determine if the column is 

1002 spatial, temporal, or neither. 

1003 sort_first : `list` [`str`] 

1004 The names of columns that should be sorted first, before spatial and 

1005 temporal columns. 

1006 

1007 Returns 

1008 ------- 

1009 `astropy.table.Table` 

1010 For convenience, the table that has been sorted. 

1011 """ 

1012 # For sorting we want to ignore the id 

1013 # We also want to move temporal or spatial dimensions earlier 

1014 sort_first = sort_first or [] 

1015 sort_early = [] 

1016 sort_late = [] 

1017 for dim in dimensions: 

1018 if dim.spatial or dim.temporal: 

1019 sort_early.extend(dim.required.names) 

1020 else: 

1021 sort_late.append(str(dim)) 

1022 sort_keys = sort_first + sort_early + sort_late 

1023 # The required names above means that we have the possibility of 

1024 # repeats of sort keys. Now have to remove them 

1025 # (order is retained by dict creation). 

1026 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1027 

1028 table.sort(sort_keys) 

1029 return table 

1030 

1031 

1032def catch_and_exit(func): 

1033 """Decorator which catches all exceptions, prints an exception traceback 

1034 and signals click to exit. 

1035 """ 

1036 

1037 @wraps(func) 

1038 def inner(*args, **kwargs): 

1039 try: 

1040 func(*args, **kwargs) 

1041 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1042 # this is handled by click itself 

1043 raise 

1044 except Exception: 

1045 exc_type, exc_value, exc_tb = sys.exc_info() 

1046 if exc_tb.tb_next: 

1047 # do not show this decorator in traceback 

1048 exc_tb = exc_tb.tb_next 

1049 log.exception( 

1050 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1051 ) 

1052 # tell click to stop, this never returns. 

1053 click.get_current_context().exit(1) 

1054 

1055 return inner