Coverage for python/lsst/daf/butler/cli/utils.py: 35%

351 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-13 10:57 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "astropyTablesToStr", 

31 "printAstropyTables", 

32 "textTypeStr", 

33 "LogCliRunner", 

34 "clickResultMsg", 

35 "command_test_env", 

36 "addArgumentHelp", 

37 "split_commas", 

38 "split_kv", 

39 "to_upper", 

40 "unwrap", 

41 "option_section", 

42 "MWPath", 

43 "MWOption", 

44 "MWArgument", 

45 "OptionSection", 

46 "MWOptionDecorator", 

47 "MWArgumentDecorator", 

48 "MWCommand", 

49 "ButlerCommand", 

50 "OptionGroup", 

51 "MWCtxObj", 

52 "yaml_presets", 

53 "sortAstropyTable", 

54 "catch_and_exit", 

55) 

56 

57 

58import itertools 

59import logging 

60import os 

61import re 

62import sys 

63import textwrap 

64import traceback 

65import uuid 

66import warnings 

67from collections import Counter 

68from collections.abc import Callable, Iterable, Iterator 

69from contextlib import contextmanager 

70from functools import partial, wraps 

71from typing import TYPE_CHECKING, Any, cast 

72from unittest.mock import patch 

73 

74import click 

75import click.exceptions 

76import click.testing 

77import yaml 

78from lsst.utils.iteration import ensure_iterable 

79 

80from .._config import Config 

81from .cliLog import CliLog 

82 

83if TYPE_CHECKING: 

84 from astropy.table import Table 

85 from lsst.daf.butler import Dimension 

86 

87log = logging.getLogger(__name__) 

88 

89# This is used as the metavar argument to Options that accept multiple string 

90# inputs, which may be comma-separarated. For example: 

91# --my-opt foo,bar --my-opt baz. 

92# Other arguments to the Option should include multiple=true and 

93# callback=split_kv. 

94typeStrAcceptsMultiple = "TEXT ..." 

95typeStrAcceptsSingle = "TEXT" 

96 

97# The standard help string for the --where option when it takes a WHERE clause. 

98where_help = ( 

99 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

100 "dimension table or a dimension name as a shortcut for the primary key column of a " 

101 "dimension table." 

102) 

103 

104 

105def astropyTablesToStr(tables: list[Table]) -> str: 

106 """Render astropy tables to string as they are displayed in the CLI. 

107 

108 Output formatting matches ``printAstropyTables``. 

109 

110 Parameters 

111 ---------- 

112 tables : `list` of `astropy.table.Table` 

113 The tables to format. 

114 

115 Returns 

116 ------- 

117 formatted : `str` 

118 Tables formatted into a string. 

119 """ 

120 ret = "" 

121 for table in tables: 

122 ret += "\n" 

123 table.pformat_all() 

124 ret += "\n" 

125 return ret 

126 

127 

128def printAstropyTables(tables: list[Table]) -> None: 

129 """Print astropy tables to be displayed in the CLI. 

130 

131 Output formatting matches ``astropyTablesToStr``. 

132 

133 Parameters 

134 ---------- 

135 tables : `list` of `astropy.table.Table` 

136 The tables to print. 

137 """ 

138 for table in tables: 

139 print("") 

140 table.pprint_all() 

141 print("") 

142 

143 

144def textTypeStr(multiple: bool) -> str: 

145 """Get the text type string for CLI help documentation. 

146 

147 Parameters 

148 ---------- 

149 multiple : `bool` 

150 True if multiple text values are allowed, False if only one value is 

151 allowed. 

152 

153 Returns 

154 ------- 

155 textTypeStr : `str` 

156 The type string to use. 

157 """ 

158 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

159 

160 

161class LogCliRunner(click.testing.CliRunner): 

162 """A test runner to use when the logging system will be initialized by code 

163 under test, calls CliLog.resetLog(), which undoes any logging setup that 

164 was done with the CliLog interface. 

165 

166 lsst.log modules can not be set back to an uninitialized state (python 

167 logging modules can be set back to NOTSET), instead they are set to 

168 `CliLog.defaultLsstLogLevel`. 

169 """ 

170 

171 def invoke(self, *args: Any, **kwargs: Any) -> Any: 

172 result = super().invoke(*args, **kwargs) 

173 CliLog.resetLog() 

174 return result 

175 

176 

177def clickResultMsg(result: click.testing.Result) -> str: 

178 """Get a standard assert message from a click result. 

179 

180 Parameters 

181 ---------- 

182 result : click.testing.Result 

183 The result object returned from `click.testing.CliRunner.invoke`. 

184 

185 Returns 

186 ------- 

187 msg : `str` 

188 The message string. 

189 """ 

190 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

191 if result.exception: 

192 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

193 return msg 

194 

195 

196@contextmanager 

197def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]: 

198 """Context manager that creates (and then cleans up) an environment that 

199 provides a CLI plugin command with the given name. 

200 

201 Parameters 

202 ---------- 

203 runner : click.testing.CliRunner 

204 The test runner to use to create the isolated filesystem. 

205 commandModule : `str` 

206 The importable module that the command can be imported from. 

207 commandName : `str` 

208 The name of the command being published to import. 

209 """ 

210 with runner.isolated_filesystem(): 

211 with open("resources.yaml", "w") as f: 

212 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

213 # Add a colon to the end of the path on the next line, this tests the 

214 # case where the lookup in LoaderCLI._getPluginList generates an empty 

215 # string in one of the list entries and verifies that the empty string 

216 # is properly stripped out. 

217 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

218 yield 

219 

220 

221def addArgumentHelp(doc: str | None, helpText: str) -> str: 

222 """Add a Click argument's help message to a function's documentation. 

223 

224 This is needed because click presents arguments in the order the argument 

225 decorators are applied to a function, top down. But, the evaluation of the 

226 decorators happens bottom up, so if arguments just append their help to the 

227 function's docstring, the argument descriptions appear in reverse order 

228 from the order they are applied in. 

229 

230 Parameters 

231 ---------- 

232 doc : `str` 

233 The function's docstring. 

234 helpText : `str` 

235 The argument's help string to be inserted into the function's 

236 docstring. 

237 

238 Returns 

239 ------- 

240 doc : `str` 

241 Updated function documentation. 

242 """ 

243 if doc is None: 243 ↛ 244line 243 didn't jump to line 244, because the condition on line 243 was never true

244 doc = helpText 

245 else: 

246 # See click documentation for details: 

247 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

248 # In short, text for the click command help can be truncated by putting 

249 # "\f" in the docstring, everything after it should be removed 

250 if "\f" in doc: 250 ↛ 251line 250 didn't jump to line 251, because the condition on line 250 was never true

251 doc = doc.split("\f")[0] 

252 

253 doclines = doc.splitlines() 

254 # The function's docstring may span multiple lines, so combine the 

255 # docstring from all the first lines until a blank line is encountered. 

256 # (Lines after the first blank line will be argument help.) 

257 while len(doclines) > 1 and doclines[1]: 

258 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

259 # Add standard indent to help text for proper alignment with command 

260 # function documentation: 

261 helpText = " " + helpText 

262 doclines.insert(1, helpText) 

263 doclines.insert(1, "\n") 

264 doc = "\n".join(doclines) 

265 return doc 

266 

267 

268def split_commas( 

269 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None 

270) -> tuple[str, ...]: 

271 """Process a tuple of values, where each value may contain comma-separated 

272 values, and return a single list of all the passed-in values. 

273 

274 This function can be passed to the 'callback' argument of a click.option to 

275 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If 

276 the comma is inside ``[]`` there will be no splitting. 

277 

278 Parameters 

279 ---------- 

280 context : `click.Context` or `None` 

281 The current execution context. Unused, but Click always passes it to 

282 callbacks. 

283 param : `click.core.Option` or `None` 

284 The parameter being handled. Unused, but Click always passes it to 

285 callbacks. 

286 values : iterable of `str` or `str` 

287 All the values passed for this option. Strings may contain commas, 

288 which will be treated as delimiters for separate values unless they 

289 are within ``[]``. 

290 

291 Returns 

292 ------- 

293 results : `tuple` [`str`] 

294 The passed in values separated by commas where appropriate and 

295 combined into a single tuple. 

296 """ 

297 if values is None: 

298 return () 

299 valueList = [] 

300 for value in ensure_iterable(values): 

301 # If we have [, or ,] we do the slow split. If square brackets 

302 # are not matching then that is likely a typo that should result 

303 # in a warning. 

304 opens = "[" 

305 closes = "]" 

306 if re.search(rf"\{opens}.*,|,.*\{closes}", value): 

307 in_parens = False 

308 current = "" 

309 for c in value: 

310 if c == opens: 

311 if in_parens: 

312 warnings.warn( 

313 f"Found second opening {opens} without corresponding closing {closes}" 

314 f" in {value!r}", 

315 stacklevel=2, 

316 ) 

317 in_parens = True 

318 elif c == closes: 

319 if not in_parens: 

320 warnings.warn( 

321 f"Found second closing {closes} without corresponding open {opens} in {value!r}", 

322 stacklevel=2, 

323 ) 

324 in_parens = False 

325 elif c == "," and not in_parens: 

326 # Split on this comma. 

327 valueList.append(current) 

328 current = "" 

329 continue 

330 current += c 

331 if in_parens: 

332 warnings.warn( 

333 f"Found opening {opens} that was never closed in {value!r}", 

334 stacklevel=2, 

335 ) 

336 if current: 

337 valueList.append(current) 

338 else: 

339 # Use efficient split since no parens. 

340 valueList.extend(value.split(",")) 

341 return tuple(valueList) 

342 

343 

344def split_kv( 

345 context: click.Context, 

346 param: click.core.Option, 

347 values: list[str], 

348 *, 

349 choice: click.Choice | None = None, 

350 multiple: bool = True, 

351 normalize: bool = False, 

352 separator: str = "=", 

353 unseparated_okay: bool = False, 

354 return_type: type[dict] | type[tuple] = dict, 

355 default_key: str = "", 

356 reverse_kv: bool = False, 

357 add_to_default: bool = False, 

358) -> dict[str, str] | tuple[tuple[str, str], ...]: 

359 """Process a tuple of values that are key-value pairs separated by a given 

360 separator. Multiple pairs may be comma separated. Return a dictionary of 

361 all the passed-in values. 

362 

363 This function can be passed to the 'callback' argument of a click.option to 

364 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

365 

366 Parameters 

367 ---------- 

368 context : `click.Context` or `None` 

369 The current execution context. Unused, but Click always passes it to 

370 callbacks. 

371 param : `click.core.Option` or `None` 

372 The parameter being handled. Unused, but Click always passes it to 

373 callbacks. 

374 values : [`str`] 

375 All the values passed for this option. Strings may contain commas, 

376 which will be treated as delimiters for separate values. 

377 choice : `click.Choice`, optional 

378 If provided, verify each value is a valid choice using the provided 

379 `click.Choice` instance. If None, no verification will be done. By 

380 default `None`. 

381 multiple : `bool`, optional 

382 If true, the value may contain multiple comma-separated values. By 

383 default True. 

384 normalize : `bool`, optional 

385 If True and `choice.case_sensitive == False`, normalize the string the 

386 user provided to match the choice's case. By default `False`. 

387 separator : str, optional 

388 The character that separates key-value pairs. May not be a comma or an 

389 empty space (for space separators use Click's default implementation 

390 for tuples; `type=(str, str)`). By default "=". 

391 unseparated_okay : `bool`, optional 

392 If True, allow values that do not have a separator. They will be 

393 returned in the values dict as a tuple of values in the key '', that 

394 is: `values[''] = (unseparated_values, )`. By default False. 

395 return_type : `type`, must be `dict` or `tuple` 

396 The type of the value that should be returned. 

397 If `dict` then the returned object will be a dict, for each item in 

398 values, the value to the left of the separator will be the key and the 

399 value to the right of the separator will be the value. 

400 If `tuple` then the returned object will be a tuple. Each item in the 

401 tuple will be 2-item tuple, the first item will be the value to the 

402 left of the separator and the second item will be the value to the 

403 right. By default `dict`. 

404 default_key : `Any` 

405 The key to use if a value is passed that is not a key-value pair. 

406 (Passing values that are not key-value pairs requires 

407 ``unseparated_okay`` to be `True`). 

408 reverse_kv : bool 

409 If true then for each item in values, the value to the left of the 

410 separator is treated as the value and the value to the right of the 

411 separator is treated as the key. By default `False`. 

412 add_to_default : `bool`, optional 

413 If True, then passed-in values will not overwrite the default value 

414 unless the ``return_type`` is `dict` and passed-in value(s) have the 

415 same key(s) as the default value. 

416 

417 Returns 

418 ------- 

419 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...] 

420 The passed-in values in dict form or tuple form. 

421 

422 Raises 

423 ------ 

424 `click.ClickException` 

425 Raised if the separator is not found in an entry, or if duplicate keys 

426 are encountered. 

427 """ 

428 

429 def norm(val: str) -> str: 

430 """If `normalize` is `True` and `choice` is not `None`, find the value 

431 in the available choices and return the value as spelled in the 

432 choices. 

433 

434 Assumes that val exists in choices; `split_kv` uses the `choice` 

435 instance to verify val is a valid choice. 

436 

437 Parameters 

438 ---------- 

439 val : `str` 

440 Value to be found. 

441 

442 Returns 

443 ------- 

444 val : `str` 

445 The value that was found or the value that was given. 

446 """ 

447 if normalize and choice is not None: 

448 v = val.casefold() 

449 for opt in choice.choices: 

450 if opt.casefold() == v: 

451 return opt 

452 return val 

453 

454 class RetDict: 

455 def __init__(self) -> None: 

456 self.ret: dict[str, str] = {} 

457 

458 def add(self, key: str, val: str) -> None: 

459 if reverse_kv: 

460 key, val = val, key 

461 self.ret[key] = val 

462 

463 def get(self) -> dict[str, str]: 

464 return self.ret 

465 

466 class RetTuple: 

467 def __init__(self) -> None: 

468 self.ret: list[tuple[str, str]] = [] 

469 

470 def add(self, key: str, val: str) -> None: 

471 if reverse_kv: 

472 key, val = val, key 

473 self.ret.append((key, val)) 

474 

475 def get(self) -> tuple[tuple[str, str], ...]: 

476 return tuple(self.ret) 

477 

478 if separator in (",", " "): 

479 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

480 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below. 

481 

482 if add_to_default: 

483 default = param.get_default(context) 

484 if default: 

485 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy 

486 

487 ret: RetDict | RetTuple 

488 if return_type is dict: 

489 ret = RetDict() 

490 elif return_type is tuple: 

491 ret = RetTuple() 

492 else: 

493 raise click.ClickException( 

494 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

495 ) 

496 if multiple: 

497 vals = split_commas(context, param, vals) 

498 for val in ensure_iterable(vals): 

499 if unseparated_okay and separator not in val: 

500 if choice is not None: 

501 choice(val) # will raise if val is an invalid choice 

502 ret.add(default_key, norm(val)) 

503 else: 

504 try: 

505 k, v = val.split(separator) 

506 if choice is not None: 

507 choice(v) # will raise if val is an invalid choice 

508 except ValueError as e: 

509 raise click.ClickException( 

510 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

511 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}" 

512 ) from None 

513 ret.add(k, norm(v)) 

514 return ret.get() 

515 

516 

517def to_upper(context: click.Context, param: click.core.Option, value: str) -> str: 

518 """Convert a value to upper case. 

519 

520 Parameters 

521 ---------- 

522 context : `click.Context` 

523 Context given by Click. 

524 param : `click.core.Option` 

525 Provided by Click. Ignored. 

526 value : `str` 

527 The value to be converted. 

528 

529 Returns 

530 ------- 

531 string 

532 A copy of the passed-in value, converted to upper case. 

533 """ 

534 return value.upper() 

535 

536 

537def unwrap(val: str) -> str: 

538 """Remove newlines and leading whitespace from a multi-line string with 

539 a consistent indentation level. 

540 

541 The first line of the string may be only a newline or may contain text 

542 followed by a newline, either is ok. After the first line, each line must 

543 begin with a consistant amount of whitespace. So, content of a 

544 triple-quoted string may begin immediately after the quotes, or the string 

545 may start with a newline. Each line after that must be the same amount of 

546 indentation/whitespace followed by text and a newline. The last line may 

547 end with a new line but is not required to do so. 

548 

549 Parameters 

550 ---------- 

551 val : `str` 

552 The string to change. 

553 

554 Returns 

555 ------- 

556 strippedString : `str` 

557 The string with newlines, indentation, and leading and trailing 

558 whitespace removed. 

559 """ 

560 

561 def splitSection(val: str) -> str: 

562 if not val.startswith("\n"): 562 ↛ 566line 562 didn't jump to line 566, because the condition on line 562 was never false

563 firstLine, _, val = val.partition("\n") 

564 firstLine += " " 

565 else: 

566 firstLine = "" 

567 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

568 

569 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

570 

571 

572class option_section: # noqa: N801 

573 """Decorator to add a section label between options in the help text of a 

574 command. 

575 

576 Parameters 

577 ---------- 

578 sectionText : `str` 

579 The text to print in the section identifier. 

580 """ 

581 

582 def __init__(self, sectionText: str) -> None: 

583 self.sectionText = "\n" + sectionText 

584 

585 def __call__(self, f: Any) -> click.Option: 

586 # Generate a parameter declaration that will be unique for this 

587 # section. 

588 return click.option( 

589 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

590 )(f) 

591 

592 

593class MWPath(click.Path): 

594 """Overrides click.Path to implement file-does-not-exist checking. 

595 

596 Changes the definition of ``exists`` so that `True` indicates the location 

597 (file or directory) must exist, `False` indicates the location must *not* 

598 exist, and `None` indicates that the file may exist or not. The standard 

599 definition for the `click.Path` ``exists`` parameter is that for `True` a 

600 location must exist, but `False` means it is not required to exist (not 

601 that it is required to not exist). 

602 

603 Parameters 

604 ---------- 

605 exists : `bool` or `None`, optional 

606 If `True`, the location (file or directory) indicated by the caller 

607 must exist. If `False` the location must not exist. If `None`, the 

608 location may exist or not. 

609 file_okay : `bool`, optional 

610 Allow a file as a value. 

611 dir_okay : `bool`, optional 

612 Allow a directory as a value. 

613 writable : `bool`, optional 

614 If `True`, a writable check is performed. 

615 readable : `bool, optional 

616 If `True`, a readable check is performed. 

617 resolve_path : `bool`, optional 

618 Resolve the path. 

619 allow_dash : `bool`, optional 

620 Allow single dash as value to mean a standard stream. 

621 path_type : `type` or `None`, optional 

622 Convert the incoming value to this type. 

623 

624 Notes 

625 ----- 

626 All parameters other than ``exists`` come directly from `click.Path`. 

627 """ 

628 

629 def __init__( 

630 self, 

631 exists: bool | None = None, 

632 file_okay: bool = True, 

633 dir_okay: bool = True, 

634 writable: bool = False, 

635 readable: bool = True, 

636 resolve_path: bool = False, 

637 allow_dash: bool = False, 

638 path_type: type | None = None, 

639 ): 

640 self.mustNotExist = exists is False 

641 if exists is None: 641 ↛ 643line 641 didn't jump to line 643, because the condition on line 641 was never false

642 exists = False 

643 super().__init__( 

644 exists=exists, 

645 file_okay=file_okay, 

646 dir_okay=dir_okay, 

647 writable=writable, 

648 readable=readable, 

649 resolve_path=resolve_path, 

650 allow_dash=allow_dash, 

651 path_type=path_type, 

652 ) 

653 

654 def convert( 

655 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None 

656 ) -> Any: 

657 """Convert values through types. 

658 

659 Called by `click.ParamType` to "convert values through types". 

660 `click.Path` uses this step to verify Path conditions. 

661 

662 Parameters 

663 ---------- 

664 value : `str` or `os.PathLike` 

665 File path. 

666 param : `click.Parameter` 

667 Parameters provided by Click. 

668 ctx : `click.Context` 

669 Context provided by Click. 

670 """ 

671 if self.mustNotExist and os.path.exists(value): 

672 self.fail(f'Path "{value}" should not exist.') 

673 return super().convert(value, param, ctx) 

674 

675 

676class MWOption(click.Option): 

677 """Overrides click.Option with desired behaviors.""" 

678 

679 def make_metavar(self) -> str: 

680 """Make the metavar for the help menu. 

681 

682 Overrides `click.Option.make_metavar`. 

683 Adds a space and an ellipsis after the metavar name if 

684 the option accepts multiple inputs, otherwise defers to the base 

685 implementation. 

686 

687 By default click does not add an ellipsis when multiple is True and 

688 nargs is 1. And when nargs does not equal 1 click adds an ellipsis 

689 without a space between the metavar and the ellipsis, but we prefer a 

690 space between. 

691 

692 Does not get called for some option types (e.g. flag) so metavar 

693 transformation that must apply to all types should be applied in 

694 get_help_record. 

695 """ 

696 metavar = super().make_metavar() 

697 if self.multiple and self.nargs == 1: 

698 metavar += " ..." 

699 elif self.nargs != 1: 

700 metavar = f"{metavar[:-3]} ..." 

701 return metavar 

702 

703 

704class MWArgument(click.Argument): 

705 """Overrides click.Argument with desired behaviors.""" 

706 

707 def make_metavar(self) -> str: 

708 """Make the metavar for the help menu. 

709 

710 Overrides `click.Option.make_metavar`. 

711 Always adds a space and an ellipsis (' ...') after the 

712 metavar name if the option accepts multiple inputs. 

713 

714 By default click adds an ellipsis without a space between the metavar 

715 and the ellipsis, but we prefer a space between. 

716 

717 Returns 

718 ------- 

719 metavar : `str` 

720 The metavar value. 

721 """ 

722 metavar = super().make_metavar() 

723 if self.nargs != 1: 

724 metavar = f"{metavar[:-3]} ..." 

725 return metavar 

726 

727 

728class OptionSection(MWOption): 

729 """Implements an Option that prints a section label in the help text and 

730 does not pass any value to the command function. 

731 

732 This class does a bit of hackery to add a section label to a click command 

733 help output: first, `expose_value` is set to `False` so that no value is 

734 passed to the command function. Second, this class overrides 

735 `click.Option.get_help_record` to return the section label string without 

736 any prefix so that it stands out as a section label. 

737 

738 This class overrides the hidden attribute because our documentation build 

739 tool, sphinx-click, implements its own `get_help_record` function which 

740 builds the record from other option values (e.g. `name`, `opts`), which 

741 breaks the hack we use to make `get_help_record` only return the 

742 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

743 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

744 entering its `_get_help_record` function. So, making the hidden property 

745 return True hides this option from sphinx-click, while allowing the section 

746 text to be returned by our `get_help_record` method when using Click. 

747 

748 The intention for this implementation is to do minimally invasive overrides 

749 of the click classes so as to be robust and easy to fix if the click 

750 internals change. 

751 

752 Parameters 

753 ---------- 

754 sectionName : `str` 

755 The parameter declaration for this option. It is not shown to the user, 

756 it must be unique within the command. If using the `section` decorator 

757 to add a section to a command's options, the section name is 

758 auto-generated. 

759 sectionText : `str` 

760 The text to print in the section identifier. 

761 """ 

762 

763 @property 

764 def hidden(self) -> bool: 

765 return True 

766 

767 @hidden.setter 

768 def hidden(self, val: Any) -> None: 

769 pass 

770 

771 def __init__(self, sectionName: str, sectionText: str) -> None: 

772 super().__init__(sectionName, expose_value=False) 

773 self.sectionText = sectionText 

774 

775 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]: 

776 return (self.sectionText, "") 

777 

778 

779class MWOptionDecorator: 

780 """Wraps the click.option decorator to enable shared options to be declared 

781 and allows inspection of the shared option. 

782 

783 Parameters 

784 ---------- 

785 *param_decls : `typing.Any` 

786 Parameters to be stored in the option. 

787 **kwargs : `typing.Any` 

788 Keyword arguments for the option. 

789 """ 

790 

791 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

792 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

793 opt = click.Option(param_decls, **kwargs) 

794 self._name = opt.name 

795 self._opts = opt.opts 

796 

797 def name(self) -> str: 

798 """Get the name that will be passed to the command function for this 

799 option. 

800 """ 

801 return cast(str, self._name) 

802 

803 def opts(self) -> list[str]: 

804 """Get the flags that will be used for this option on the command 

805 line. 

806 """ 

807 return self._opts 

808 

809 @property 

810 def help(self) -> str: 

811 """Get the help text for this option. Returns an empty string if no 

812 help was defined. 

813 """ 

814 return self.partialOpt.keywords.get("help", "") 

815 

816 def __call__(self, *args: Any, **kwargs: Any) -> Any: 

817 return self.partialOpt(*args, **kwargs) 

818 

819 

820class MWArgumentDecorator: 

821 """Wraps the click.argument decorator to enable shared arguments to be 

822 declared. 

823 

824 Parameters 

825 ---------- 

826 *param_decls : `typing.Any` 

827 Parameters to be stored in the argument. 

828 **kwargs : `typing.Any` 

829 Keyword arguments for the argument. 

830 """ 

831 

832 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

833 self._helpText = kwargs.pop("help", None) 

834 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

835 

836 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable: 

837 def decorator(f: Any) -> Any: 

838 if help is not None: 

839 self._helpText = help 

840 if self._helpText: 

841 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

842 return self.partialArg(*args, **kwargs)(f) 

843 

844 return decorator 

845 

846 

847class MWCommand(click.Command): 

848 """Command subclass that stores a copy of the args list for use by the 

849 command. 

850 

851 Parameters 

852 ---------- 

853 *args : `typing.Any` 

854 Arguments for `click.Command`. 

855 **kwargs : `typing.Any` 

856 Keyword arguments for `click.Command`. 

857 """ 

858 

859 extra_epilog: str | None = None 

860 

861 def __init__(self, *args: Any, **kwargs: Any) -> None: 

862 # wrap callback method with catch_and_exit decorator 

863 callback = kwargs.get("callback") 

864 if callback is not None: 864 ↛ 867line 864 didn't jump to line 867, because the condition on line 864 was never false

865 kwargs = kwargs.copy() 

866 kwargs["callback"] = catch_and_exit(callback) 

867 super().__init__(*args, **kwargs) 

868 

869 def _capture_args(self, ctx: click.Context, args: list[str]) -> None: 

870 """Capture the command line options and arguments. 

871 

872 See details about what is captured and the order in which it is stored 

873 in the documentation of `MWCtxObj`. 

874 

875 Parameters 

876 ---------- 

877 ctx : `click.Context` 

878 The current Context. 

879 args : `list` [`str`] 

880 The list of arguments from the command line, split at spaces but 

881 not at separators (like "="). 

882 """ 

883 parser = self.make_parser(ctx) 

884 opts, _, param_order = parser.parse_args(args=list(args)) 

885 # `param_order` is a list of click.Option and click.Argument, there is 

886 # one item for each time the Option or Argument was used on the 

887 # command line. Options will precede Arguments, within each sublist 

888 # they are in the order they were used on the command line. Note that 

889 # click.Option and click.Argument do not contain the value from the 

890 # command line; values are in `opts`. 

891 # 

892 # `opts` is a dict where the key is the argument name to the 

893 # click.Command function, this name matches the `click.Option.name` or 

894 # `click.Argument.name`. For Options, an item will only be present if 

895 # the Option was used on the command line. For Arguments, an item will 

896 # always be present and if no value was provided on the command line 

897 # the value will be `None`. If the option accepts multiple values, the 

898 # value in `opts` is a tuple, otherwise it is a single item. 

899 next_idx: Counter = Counter() 

900 captured_args = [] 

901 for param in param_order: 

902 if isinstance(param, click.Option): 

903 param_name = cast(str, param.name) 

904 if param.multiple: 

905 val = opts[param_name][next_idx[param_name]] 

906 next_idx[param_name] += 1 

907 else: 

908 val = opts[param_name] 

909 if param.is_flag: 

910 # Bool options store their True flags in opts and their 

911 # False flags in secondary_opts. 

912 if val: 

913 flag = max(param.opts, key=len) 

914 else: 

915 flag = max(param.secondary_opts, key=len) 

916 captured_args.append(flag) 

917 else: 

918 captured_args.append(max(param.opts, key=len)) 

919 captured_args.append(val) 

920 elif isinstance(param, click.Argument): 

921 param_name = cast(str, param.name) 

922 if (opt := opts[param_name]) is not None: 

923 captured_args.append(opt) 

924 else: 

925 raise AssertionError("All parameters should be an Option or an Argument") 

926 MWCtxObj.getFrom(ctx).args = captured_args 

927 

928 def parse_args(self, ctx: click.Context, args: Any) -> list[str]: 

929 """Given a context and a list of arguments this creates the parser and 

930 parses the arguments, then modifies the context as necessary. This is 

931 automatically invoked by make_context(). 

932 

933 This function overrides `click.Command.parse_args`. 

934 

935 The call to `_capture_args` in this override stores the arguments 

936 (option names, option value, and argument values) that were used by the 

937 caller on the command line in the context object. These stored 

938 arugments can be used by the command function, e.g. to process options 

939 in the order they appeared on the command line (pipetask uses this 

940 feature to create pipeline actions in an order from different options). 

941 

942 Parameters 

943 ---------- 

944 ctx : `click.core.Context` 

945 The current Context. 

946 args : `list` [`str`] 

947 The list of arguments from the command line, split at spaces but 

948 not at separators (like "="). 

949 """ 

950 self._capture_args(ctx, args) 

951 return super().parse_args(ctx, args) 

952 

953 @property 

954 def epilog(self) -> str | None: 

955 """Override the epilog attribute to add extra_epilog (if defined by a 

956 subclass) to the end of any epilog provided by a subcommand. 

957 """ 

958 ret = self._epilog if self._epilog else "" 

959 if self.extra_epilog: 

960 if ret: 

961 ret += "\n\n" 

962 ret += self.extra_epilog 

963 return ret 

964 

965 @epilog.setter 

966 def epilog(self, val: str) -> None: 

967 self._epilog = val 

968 

969 

970class ButlerCommand(MWCommand): 

971 """Command subclass with butler-command specific overrides.""" 

972 

973 extra_epilog = "See 'butler --help' for more options." 

974 

975 

976class OptionGroup: 

977 """Base class for an option group decorator. Requires the option group 

978 subclass to have a property called `decorator`. 

979 """ 

980 

981 decorators: list[Any] 

982 

983 def __call__(self, f: Any) -> Any: 

984 for decorator in reversed(self.decorators): 

985 f = decorator(f) 

986 return f 

987 

988 

989class MWCtxObj: 

990 """Helper object for managing the `click.Context.obj` parameter, allows 

991 obj data to be managed in a consistent way. 

992 

993 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

994 initialize the obj if needed and return a new or existing `MWCtxObj`. 

995 

996 The `args` attribute contains a list of options, option values, and 

997 argument values that is similar to the list of arguments and options that 

998 were passed in on the command line, with differences noted below: 

999 

1000 * Option namess and option values are first in the list, and argument 

1001 values come last. The order of options and option values is preserved 

1002 within the options. The order of argument values is preserved. 

1003 

1004 * The longest option name is used for the option in the `args` list, e.g. 

1005 if an option accepts both short and long names "-o / --option" and the 

1006 short option name "-o" was used on the command line, the longer name will 

1007 be the one that appears in `args`. 

1008 

1009 * A long option name (which begins with two dashes "--") and its value may 

1010 be separated by an equal sign; the name and value are split at the equal 

1011 sign and it is removed. In `args`, the option is in one list item, and 

1012 the option value (without the equal sign) is in the next list item. e.g. 

1013 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

1014 `args`. 

1015 

1016 * A short option name, (which begins with one dash "-") and its value are 

1017 split immediately after the short option name, and if there is 

1018 whitespace between the short option name and its value it is removed. 

1019 Everything after the short option name (excluding whitespace) is included 

1020 in the value. If the `Option` has a long name, the long name will be used 

1021 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

1022 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

1023 `["--option", "=foo"]` (because everything after the short option name, 

1024 except whitespace, is used for the value (as is standard with unix 

1025 command line tools). 

1026 

1027 Attributes 

1028 ---------- 

1029 args : `list` [`str`] 

1030 A list of options, option values, and arguments simialr to those that 

1031 were passed in on the command line. See comments about captured options 

1032 & arguments above. 

1033 """ 

1034 

1035 def __init__(self) -> None: 

1036 self.args = None 

1037 

1038 @staticmethod 

1039 def getFrom(ctx: click.Context) -> Any: 

1040 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

1041 the new or already existing `MWCtxObj`. 

1042 

1043 Parameters 

1044 ---------- 

1045 ctx : `click.Context` 

1046 Context provided by Click. 

1047 """ 

1048 if ctx.obj is not None: 

1049 return ctx.obj 

1050 ctx.obj = MWCtxObj() 

1051 return ctx.obj 

1052 

1053 

1054def yaml_presets(ctx: click.Context, param: str, value: Any) -> None: 

1055 """Read additional values from the supplied YAML file. 

1056 

1057 Parameters 

1058 ---------- 

1059 ctx : `click.context` 

1060 The context for the click operation. Used to extract the subcommand 

1061 name and translate option & argument names. 

1062 param : `str` 

1063 The parameter name. 

1064 value : `object` 

1065 The value of the parameter. 

1066 """ 

1067 

1068 def _name_for_option(ctx: click.Context, option: str) -> str: 

1069 """Use a CLI option name to find the name of the argument to the 

1070 command function. 

1071 

1072 Parameters 

1073 ---------- 

1074 ctx : `click.Context` 

1075 The context for the click operation. 

1076 option : `str` 

1077 The option/argument name from the yaml file. 

1078 

1079 Returns 

1080 ------- 

1081 name : str 

1082 The name of the argument to use when calling the click.command 

1083 function, as it should appear in the `ctx.default_map`. 

1084 

1085 Raises 

1086 ------ 

1087 RuntimeError 

1088 Raised if the option name from the yaml file does not exist in the 

1089 command parameters. This catches misspellings and incorrect useage 

1090 in the yaml file. 

1091 """ 

1092 for param in ctx.command.params: 

1093 # Remove leading dashes: they are not used for option names in the 

1094 # yaml file. 

1095 if option in [opt.lstrip("-") for opt in param.opts]: 

1096 return cast(str, param.name) 

1097 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

1098 

1099 ctx.default_map = ctx.default_map or {} 

1100 cmd_name = ctx.info_name 

1101 assert cmd_name is not None, "command name cannot be None" 

1102 if value: 

1103 try: 

1104 overrides = _read_yaml_presets(value, cmd_name) 

1105 options = list(overrides.keys()) 

1106 for option in options: 

1107 name = _name_for_option(ctx, option) 

1108 if name == option: 

1109 continue 

1110 overrides[name] = overrides.pop(option) 

1111 except Exception as e: 

1112 raise click.BadOptionUsage( 

1113 option_name=param, 

1114 message=f"Error reading overrides file: {e}", 

1115 ctx=ctx, 

1116 ) from None 

1117 # Override the defaults for this subcommand 

1118 ctx.default_map.update(overrides) 

1119 return 

1120 

1121 

1122def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]: 

1123 """Read file command line overrides from YAML config file. 

1124 

1125 Parameters 

1126 ---------- 

1127 file_uri : `str` 

1128 URI of override YAML file containing the command line overrides. 

1129 They should be grouped by command name. 

1130 cmd_name : `str` 

1131 The subcommand name that is being modified. 

1132 

1133 Returns 

1134 ------- 

1135 overrides : `dict` of [`str`, Any] 

1136 The relevant command line options read from the override file. 

1137 """ 

1138 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

1139 config = Config(file_uri) 

1140 return config[cmd_name] 

1141 

1142 

1143def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table: 

1144 """Sort an astropy table. 

1145 

1146 Prioritization is given to columns in this order: 

1147 

1148 1. the provided named columns 

1149 2. spatial and temporal columns 

1150 3. the rest of the columns. 

1151 

1152 The table is sorted in-place, and is also returned for convenience. 

1153 

1154 Parameters 

1155 ---------- 

1156 table : `astropy.table.Table` 

1157 The table to sort. 

1158 dimensions : `list` [``Dimension``] 

1159 The dimensions of the dataIds in the table (the dimensions should be 

1160 the same for all the dataIds). Used to determine if the column is 

1161 spatial, temporal, or neither. 

1162 sort_first : `list` [`str`] 

1163 The names of columns that should be sorted first, before spatial and 

1164 temporal columns. 

1165 

1166 Returns 

1167 ------- 

1168 `astropy.table.Table` 

1169 For convenience, the table that has been sorted. 

1170 """ 

1171 # For sorting we want to ignore the id 

1172 # We also want to move temporal or spatial dimensions earlier 

1173 sort_first = sort_first or [] 

1174 sort_early: list[str] = [] 

1175 sort_late: list[str] = [] 

1176 for dim in dimensions: 

1177 if dim.spatial or dim.temporal: 

1178 sort_early.extend(dim.required.names) 

1179 else: 

1180 sort_late.append(str(dim)) 

1181 sort_keys = sort_first + sort_early + sort_late 

1182 # The required names above means that we have the possibility of 

1183 # repeats of sort keys. Now have to remove them 

1184 # (order is retained by dict creation). 

1185 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1186 

1187 table.sort(sort_keys) 

1188 return table 

1189 

1190 

1191def catch_and_exit(func: Callable) -> Callable: 

1192 """Catch all exceptions, prints an exception traceback 

1193 and signals click to exit. 

1194 

1195 Use as decorator. 

1196 

1197 Parameters 

1198 ---------- 

1199 func : `collections.abc.Callable` 

1200 The function to be decorated. 

1201 

1202 Returns 

1203 ------- 

1204 `collections.abc.Callable` 

1205 The decorated function. 

1206 """ 

1207 

1208 @wraps(func) 

1209 def inner(*args: Any, **kwargs: Any) -> None: 

1210 try: 

1211 func(*args, **kwargs) 

1212 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1213 # this is handled by click itself 

1214 raise 

1215 except Exception: 

1216 exc_type, exc_value, exc_tb = sys.exc_info() 

1217 assert exc_type is not None 

1218 assert exc_value is not None 

1219 assert exc_tb is not None 

1220 if exc_tb.tb_next: 

1221 # do not show this decorator in traceback 

1222 exc_tb = exc_tb.tb_next 

1223 log.exception( 

1224 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1225 ) 

1226 # tell click to stop, this never returns. 

1227 click.get_current_context().exit(1) 

1228 

1229 return inner