Coverage for python/lsst/daf/butler/cli/utils.py: 35%

351 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-08-12 09:20 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "astropyTablesToStr", 

25 "printAstropyTables", 

26 "textTypeStr", 

27 "LogCliRunner", 

28 "clickResultMsg", 

29 "command_test_env", 

30 "addArgumentHelp", 

31 "split_commas", 

32 "split_kv", 

33 "to_upper", 

34 "unwrap", 

35 "option_section", 

36 "MWPath", 

37 "MWOption", 

38 "MWArgument", 

39 "OptionSection", 

40 "MWOptionDecorator", 

41 "MWArgumentDecorator", 

42 "MWCommand", 

43 "ButlerCommand", 

44 "OptionGroup", 

45 "MWCtxObj", 

46 "yaml_presets", 

47 "sortAstropyTable", 

48 "catch_and_exit", 

49) 

50 

51 

52import itertools 

53import logging 

54import os 

55import re 

56import sys 

57import textwrap 

58import traceback 

59import uuid 

60import warnings 

61from collections import Counter 

62from collections.abc import Callable, Iterable, Iterator 

63from contextlib import contextmanager 

64from functools import partial, wraps 

65from typing import TYPE_CHECKING, Any, cast 

66from unittest.mock import patch 

67 

68import click 

69import click.exceptions 

70import click.testing 

71import yaml 

72from lsst.utils.iteration import ensure_iterable 

73 

74from ..core.config import Config 

75from .cliLog import CliLog 

76 

77if TYPE_CHECKING: 

78 from astropy.table import Table 

79 from lsst.daf.butler import Dimension 

80 

81log = logging.getLogger(__name__) 

82 

83# This is used as the metavar argument to Options that accept multiple string 

84# inputs, which may be comma-separarated. For example: 

85# --my-opt foo,bar --my-opt baz. 

86# Other arguments to the Option should include multiple=true and 

87# callback=split_kv. 

88typeStrAcceptsMultiple = "TEXT ..." 

89typeStrAcceptsSingle = "TEXT" 

90 

91# The standard help string for the --where option when it takes a WHERE clause. 

92where_help = ( 

93 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

94 "dimension table or a dimension name as a shortcut for the primary key column of a " 

95 "dimension table." 

96) 

97 

98 

99def astropyTablesToStr(tables: list[Table]) -> str: 

100 """Render astropy tables to string as they are displayed in the CLI. 

101 

102 Output formatting matches ``printAstropyTables``. 

103 """ 

104 ret = "" 

105 for table in tables: 

106 ret += "\n" 

107 table.pformat_all() 

108 ret += "\n" 

109 return ret 

110 

111 

112def printAstropyTables(tables: list[Table]) -> None: 

113 """Print astropy tables to be displayed in the CLI. 

114 

115 Output formatting matches ``astropyTablesToStr``. 

116 """ 

117 for table in tables: 

118 print("") 

119 table.pprint_all() 

120 print("") 

121 

122 

123def textTypeStr(multiple: bool) -> str: 

124 """Get the text type string for CLI help documentation. 

125 

126 Parameters 

127 ---------- 

128 multiple : `bool` 

129 True if multiple text values are allowed, False if only one value is 

130 allowed. 

131 

132 Returns 

133 ------- 

134 textTypeStr : `str` 

135 The type string to use. 

136 """ 

137 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

138 

139 

140class LogCliRunner(click.testing.CliRunner): 

141 """A test runner to use when the logging system will be initialized by code 

142 under test, calls CliLog.resetLog(), which undoes any logging setup that 

143 was done with the CliLog interface. 

144 

145 lsst.log modules can not be set back to an uninitialized state (python 

146 logging modules can be set back to NOTSET), instead they are set to 

147 `CliLog.defaultLsstLogLevel`. 

148 """ 

149 

150 def invoke(self, *args: Any, **kwargs: Any) -> Any: 

151 result = super().invoke(*args, **kwargs) 

152 CliLog.resetLog() 

153 return result 

154 

155 

156def clickResultMsg(result: click.testing.Result) -> str: 

157 """Get a standard assert message from a click result. 

158 

159 Parameters 

160 ---------- 

161 result : click.testing.Result 

162 The result object returned from click.testing.CliRunner.invoke 

163 

164 Returns 

165 ------- 

166 msg : `str` 

167 The message string. 

168 """ 

169 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

170 if result.exception: 

171 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

172 return msg 

173 

174 

175@contextmanager 

176def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]: 

177 """Context manager that creates (and then cleans up) an environment that 

178 provides a CLI plugin command with the given name. 

179 

180 Parameters 

181 ---------- 

182 runner : click.testing.CliRunner 

183 The test runner to use to create the isolated filesystem. 

184 commandModule : `str` 

185 The importable module that the command can be imported from. 

186 commandName : `str` 

187 The name of the command being published to import. 

188 """ 

189 with runner.isolated_filesystem(): 

190 with open("resources.yaml", "w") as f: 

191 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

192 # Add a colon to the end of the path on the next line, this tests the 

193 # case where the lookup in LoaderCLI._getPluginList generates an empty 

194 # string in one of the list entries and verifies that the empty string 

195 # is properly stripped out. 

196 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

197 yield 

198 

199 

200def addArgumentHelp(doc: str | None, helpText: str) -> str: 

201 """Add a Click argument's help message to a function's documentation. 

202 

203 This is needed because click presents arguments in the order the argument 

204 decorators are applied to a function, top down. But, the evaluation of the 

205 decorators happens bottom up, so if arguments just append their help to the 

206 function's docstring, the argument descriptions appear in reverse order 

207 from the order they are applied in. 

208 

209 Parameters 

210 ---------- 

211 doc : `str` 

212 The function's docstring. 

213 helpText : `str` 

214 The argument's help string to be inserted into the function's 

215 docstring. 

216 

217 Returns 

218 ------- 

219 doc : `str` 

220 Updated function documentation. 

221 """ 

222 if doc is None: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true

223 doc = helpText 

224 else: 

225 # See click documentation for details: 

226 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

227 # In short, text for the click command help can be truncated by putting 

228 # "\f" in the docstring, everything after it should be removed 

229 if "\f" in doc: 229 ↛ 230line 229 didn't jump to line 230, because the condition on line 229 was never true

230 doc = doc.split("\f")[0] 

231 

232 doclines = doc.splitlines() 

233 # The function's docstring may span multiple lines, so combine the 

234 # docstring from all the first lines until a blank line is encountered. 

235 # (Lines after the first blank line will be argument help.) 

236 while len(doclines) > 1 and doclines[1]: 

237 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

238 # Add standard indent to help text for proper alignment with command 

239 # function documentation: 

240 helpText = " " + helpText 

241 doclines.insert(1, helpText) 

242 doclines.insert(1, "\n") 

243 doc = "\n".join(doclines) 

244 return doc 

245 

246 

247def split_commas( 

248 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None 

249) -> tuple[str, ...]: 

250 """Process a tuple of values, where each value may contain comma-separated 

251 values, and return a single list of all the passed-in values. 

252 

253 This function can be passed to the 'callback' argument of a click.option to 

254 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If 

255 the comma is inside ``[]`` there will be no splitting. 

256 

257 Parameters 

258 ---------- 

259 context : `click.Context` or `None` 

260 The current execution context. Unused, but Click always passes it to 

261 callbacks. 

262 param : `click.core.Option` or `None` 

263 The parameter being handled. Unused, but Click always passes it to 

264 callbacks. 

265 values : iterable of `str` or `str` 

266 All the values passed for this option. Strings may contain commas, 

267 which will be treated as delimiters for separate values unless they 

268 are within ``[]``. 

269 

270 Returns 

271 ------- 

272 results : `tuple` [`str`] 

273 The passed in values separated by commas where appropriate and 

274 combined into a single tuple. 

275 """ 

276 if values is None: 

277 return () 

278 valueList = [] 

279 for value in ensure_iterable(values): 

280 # If we have [, or ,] we do the slow split. If square brackets 

281 # are not matching then that is likely a typo that should result 

282 # in a warning. 

283 opens = "[" 

284 closes = "]" 

285 if re.search(rf"\{opens}.*,|,.*\{closes}", value): 

286 in_parens = False 

287 current = "" 

288 for c in value: 

289 if c == opens: 

290 if in_parens: 

291 warnings.warn( 

292 f"Found second opening {opens} without corresponding closing {closes}" 

293 f" in {value!r}", 

294 stacklevel=2, 

295 ) 

296 in_parens = True 

297 elif c == closes: 

298 if not in_parens: 

299 warnings.warn( 

300 f"Found second closing {closes} without corresponding open {opens} in {value!r}", 

301 stacklevel=2, 

302 ) 

303 in_parens = False 

304 elif c == "," and not in_parens: 

305 # Split on this comma. 

306 valueList.append(current) 

307 current = "" 

308 continue 

309 current += c 

310 if in_parens: 

311 warnings.warn( 

312 f"Found opening {opens} that was never closed in {value!r}", 

313 stacklevel=2, 

314 ) 

315 if current: 

316 valueList.append(current) 

317 else: 

318 # Use efficient split since no parens. 

319 valueList.extend(value.split(",")) 

320 return tuple(valueList) 

321 

322 

323def split_kv( 

324 context: click.Context, 

325 param: click.core.Option, 

326 values: list[str], 

327 *, 

328 choice: click.Choice | None = None, 

329 multiple: bool = True, 

330 normalize: bool = False, 

331 separator: str = "=", 

332 unseparated_okay: bool = False, 

333 return_type: type[dict] | type[tuple] = dict, 

334 default_key: str = "", 

335 reverse_kv: bool = False, 

336 add_to_default: bool = False, 

337) -> dict[str, str] | tuple[tuple[str, str], ...]: 

338 """Process a tuple of values that are key-value pairs separated by a given 

339 separator. Multiple pairs may be comma separated. Return a dictionary of 

340 all the passed-in values. 

341 

342 This function can be passed to the 'callback' argument of a click.option to 

343 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

344 

345 Parameters 

346 ---------- 

347 context : `click.Context` or `None` 

348 The current execution context. Unused, but Click always passes it to 

349 callbacks. 

350 param : `click.core.Option` or `None` 

351 The parameter being handled. Unused, but Click always passes it to 

352 callbacks. 

353 values : [`str`] 

354 All the values passed for this option. Strings may contain commas, 

355 which will be treated as delimiters for separate values. 

356 choice : `click.Choice`, optional 

357 If provided, verify each value is a valid choice using the provided 

358 `click.Choice` instance. If None, no verification will be done. By 

359 default None 

360 multiple : `bool`, optional 

361 If true, the value may contain multiple comma-separated values. By 

362 default True. 

363 normalize : `bool`, optional 

364 If True and `choice.case_sensitive == False`, normalize the string the 

365 user provided to match the choice's case. By default False. 

366 separator : str, optional 

367 The character that separates key-value pairs. May not be a comma or an 

368 empty space (for space separators use Click's default implementation 

369 for tuples; `type=(str, str)`). By default "=". 

370 unseparated_okay : `bool`, optional 

371 If True, allow values that do not have a separator. They will be 

372 returned in the values dict as a tuple of values in the key '', that 

373 is: `values[''] = (unseparated_values, )`. By default False. 

374 return_type : `type`, must be `dict` or `tuple` 

375 The type of the value that should be returned. 

376 If `dict` then the returned object will be a dict, for each item in 

377 values, the value to the left of the separator will be the key and the 

378 value to the right of the separator will be the value. 

379 If `tuple` then the returned object will be a tuple. Each item in the 

380 tuple will be 2-item tuple, the first item will be the value to the 

381 left of the separator and the second item will be the value to the 

382 right. By default `dict`. 

383 default_key : `Any` 

384 The key to use if a value is passed that is not a key-value pair. 

385 (Passing values that are not key-value pairs requires 

386 ``unseparated_okay`` to be `True`.) 

387 reverse_kv : bool 

388 If true then for each item in values, the value to the left of the 

389 separator is treated as the value and the value to the right of the 

390 separator is treated as the key. By default False. 

391 add_to_default : `bool`, optional 

392 If True, then passed-in values will not overwrite the default value 

393 unless the ``return_type`` is `dict` and passed-in value(s) have the 

394 same key(s) as the default value. 

395 

396 Returns 

397 ------- 

398 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...] 

399 The passed-in values in dict form or tuple form. 

400 

401 Raises 

402 ------ 

403 `click.ClickException` 

404 Raised if the separator is not found in an entry, or if duplicate keys 

405 are encountered. 

406 """ 

407 

408 def norm(val: str) -> str: 

409 """If `normalize` is True and `choice` is not `None`, find the value 

410 in the available choices and return the value as spelled in the 

411 choices. 

412 

413 Assumes that val exists in choices; `split_kv` uses the `choice` 

414 instance to verify val is a valid choice. 

415 """ 

416 if normalize and choice is not None: 

417 v = val.casefold() 

418 for opt in choice.choices: 

419 if opt.casefold() == v: 

420 return opt 

421 return val 

422 

423 class RetDict: 

424 def __init__(self) -> None: 

425 self.ret: dict[str, str] = {} 

426 

427 def add(self, key: str, val: str) -> None: 

428 if reverse_kv: 

429 key, val = val, key 

430 self.ret[key] = val 

431 

432 def get(self) -> dict[str, str]: 

433 return self.ret 

434 

435 class RetTuple: 

436 def __init__(self) -> None: 

437 self.ret: list[tuple[str, str]] = [] 

438 

439 def add(self, key: str, val: str) -> None: 

440 if reverse_kv: 

441 key, val = val, key 

442 self.ret.append((key, val)) 

443 

444 def get(self) -> tuple[tuple[str, str], ...]: 

445 return tuple(self.ret) 

446 

447 if separator in (",", " "): 

448 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

449 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below. 

450 

451 if add_to_default: 

452 default = param.get_default(context) 

453 if default: 

454 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy 

455 

456 ret: RetDict | RetTuple 

457 if return_type is dict: 

458 ret = RetDict() 

459 elif return_type is tuple: 

460 ret = RetTuple() 

461 else: 

462 raise click.ClickException( 

463 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

464 ) 

465 if multiple: 

466 vals = split_commas(context, param, vals) 

467 for val in ensure_iterable(vals): 

468 if unseparated_okay and separator not in val: 

469 if choice is not None: 

470 choice(val) # will raise if val is an invalid choice 

471 ret.add(default_key, norm(val)) 

472 else: 

473 try: 

474 k, v = val.split(separator) 

475 if choice is not None: 

476 choice(v) # will raise if val is an invalid choice 

477 except ValueError as e: 

478 raise click.ClickException( 

479 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

480 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}" 

481 ) from None 

482 ret.add(k, norm(v)) 

483 return ret.get() 

484 

485 

486def to_upper(context: click.Context, param: click.core.Option, value: str) -> str: 

487 """Convert a value to upper case. 

488 

489 Parameters 

490 ---------- 

491 context : click.Context 

492 

493 values : string 

494 The value to be converted. 

495 

496 Returns 

497 ------- 

498 string 

499 A copy of the passed-in value, converted to upper case. 

500 """ 

501 return value.upper() 

502 

503 

504def unwrap(val: str) -> str: 

505 """Remove newlines and leading whitespace from a multi-line string with 

506 a consistent indentation level. 

507 

508 The first line of the string may be only a newline or may contain text 

509 followed by a newline, either is ok. After the first line, each line must 

510 begin with a consistant amount of whitespace. So, content of a 

511 triple-quoted string may begin immediately after the quotes, or the string 

512 may start with a newline. Each line after that must be the same amount of 

513 indentation/whitespace followed by text and a newline. The last line may 

514 end with a new line but is not required to do so. 

515 

516 Parameters 

517 ---------- 

518 val : `str` 

519 The string to change. 

520 

521 Returns 

522 ------- 

523 strippedString : `str` 

524 The string with newlines, indentation, and leading and trailing 

525 whitespace removed. 

526 """ 

527 

528 def splitSection(val: str) -> str: 

529 if not val.startswith("\n"): 529 ↛ 533line 529 didn't jump to line 533, because the condition on line 529 was never false

530 firstLine, _, val = val.partition("\n") 

531 firstLine += " " 

532 else: 

533 firstLine = "" 

534 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

535 

536 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

537 

538 

539class option_section: # noqa: N801 

540 """Decorator to add a section label between options in the help text of a 

541 command. 

542 

543 Parameters 

544 ---------- 

545 sectionText : `str` 

546 The text to print in the section identifier. 

547 """ 

548 

549 def __init__(self, sectionText: str) -> None: 

550 self.sectionText = "\n" + sectionText 

551 

552 def __call__(self, f: Any) -> click.Option: 

553 # Generate a parameter declaration that will be unique for this 

554 # section. 

555 return click.option( 

556 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

557 )(f) 

558 

559 

560class MWPath(click.Path): 

561 """Overrides click.Path to implement file-does-not-exist checking. 

562 

563 Changes the definition of ``exists` so that `True` indicates the location 

564 (file or directory) must exist, `False` indicates the location must *not* 

565 exist, and `None` indicates that the file may exist or not. The standard 

566 definition for the `click.Path` ``exists`` parameter is that for `True` a 

567 location must exist, but `False` means it is not required to exist (not 

568 that it is required to not exist). 

569 

570 Parameters 

571 ---------- 

572 exists : `True`, `False`, or `None` 

573 If `True`, the location (file or directory) indicated by the caller 

574 must exist. If `False` the location must not exist. If `None`, the 

575 location may exist or not. 

576 

577 For other parameters see `click.Path`. 

578 """ 

579 

580 def __init__( 

581 self, 

582 exists: bool | None = None, 

583 file_okay: bool = True, 

584 dir_okay: bool = True, 

585 writable: bool = False, 

586 readable: bool = True, 

587 resolve_path: bool = False, 

588 allow_dash: bool = False, 

589 path_type: type | None = None, 

590 ): 

591 self.mustNotExist = exists is False 

592 if exists is None: 592 ↛ 594line 592 didn't jump to line 594, because the condition on line 592 was never false

593 exists = False 

594 super().__init__( 

595 exists=exists, 

596 file_okay=file_okay, 

597 dir_okay=dir_okay, 

598 writable=writable, 

599 readable=readable, 

600 resolve_path=resolve_path, 

601 allow_dash=allow_dash, 

602 path_type=path_type, 

603 ) 

604 

605 def convert( 

606 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None 

607 ) -> Any: 

608 """Convert values through types. 

609 

610 Called by `click.ParamType` to "convert values through types". 

611 `click.Path` uses this step to verify Path conditions. 

612 """ 

613 if self.mustNotExist and os.path.exists(value): 

614 self.fail(f'Path "{value}" should not exist.') 

615 return super().convert(value, param, ctx) 

616 

617 

618class MWOption(click.Option): 

619 """Overrides click.Option with desired behaviors.""" 

620 

621 def make_metavar(self) -> str: 

622 """Make the metavar for the help menu. 

623 

624 Overrides `click.Option.make_metavar`. 

625 Adds a space and an ellipsis after the metavar name if 

626 the option accepts multiple inputs, otherwise defers to the base 

627 implementation. 

628 

629 By default click does not add an ellipsis when multiple is True and 

630 nargs is 1. And when nargs does not equal 1 click adds an ellipsis 

631 without a space between the metavar and the ellipsis, but we prefer a 

632 space between. 

633 

634 Does not get called for some option types (e.g. flag) so metavar 

635 transformation that must apply to all types should be applied in 

636 get_help_record. 

637 """ 

638 metavar = super().make_metavar() 

639 if self.multiple and self.nargs == 1: 

640 metavar += " ..." 

641 elif self.nargs != 1: 

642 metavar = f"{metavar[:-3]} ..." 

643 return metavar 

644 

645 

646class MWArgument(click.Argument): 

647 """Overrides click.Argument with desired behaviors.""" 

648 

649 def make_metavar(self) -> str: 

650 """Make the metavar for the help menu. 

651 

652 Overrides `click.Option.make_metavar`. 

653 Always adds a space and an ellipsis (' ...') after the 

654 metavar name if the option accepts multiple inputs. 

655 

656 By default click adds an ellipsis without a space between the metavar 

657 and the ellipsis, but we prefer a space between. 

658 

659 Returns 

660 ------- 

661 metavar : `str` 

662 The metavar value. 

663 """ 

664 metavar = super().make_metavar() 

665 if self.nargs != 1: 

666 metavar = f"{metavar[:-3]} ..." 

667 return metavar 

668 

669 

670class OptionSection(MWOption): 

671 """Implements an Option that prints a section label in the help text and 

672 does not pass any value to the command function. 

673 

674 This class does a bit of hackery to add a section label to a click command 

675 help output: first, `expose_value` is set to `False` so that no value is 

676 passed to the command function. Second, this class overrides 

677 `click.Option.get_help_record` to return the section label string without 

678 any prefix so that it stands out as a section label. 

679 

680 This class overrides the hidden attribute because our documentation build 

681 tool, sphinx-click, implements its own `get_help_record` function which 

682 builds the record from other option values (e.g. `name`, `opts`), which 

683 breaks the hack we use to make `get_help_record` only return the 

684 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

685 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

686 entering its `_get_help_record` function. So, making the hidden property 

687 return True hides this option from sphinx-click, while allowing the section 

688 text to be returned by our `get_help_record` method when using Click. 

689 

690 The intention for this implementation is to do minimally invasive overrides 

691 of the click classes so as to be robust and easy to fix if the click 

692 internals change. 

693 

694 Parameters 

695 ---------- 

696 sectionName : `str` 

697 The parameter declaration for this option. It is not shown to the user, 

698 it must be unique within the command. If using the `section` decorator 

699 to add a section to a command's options, the section name is 

700 auto-generated. 

701 sectionText : `str` 

702 The text to print in the section identifier. 

703 """ 

704 

705 @property 

706 def hidden(self) -> bool: 

707 return True 

708 

709 @hidden.setter 

710 def hidden(self, val: Any) -> None: 

711 pass 

712 

713 def __init__(self, sectionName: str, sectionText: str) -> None: 

714 super().__init__(sectionName, expose_value=False) 

715 self.sectionText = sectionText 

716 

717 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]: 

718 return (self.sectionText, "") 

719 

720 

721class MWOptionDecorator: 

722 """Wraps the click.option decorator to enable shared options to be declared 

723 and allows inspection of the shared option. 

724 """ 

725 

726 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

727 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

728 opt = click.Option(param_decls, **kwargs) 

729 self._name = opt.name 

730 self._opts = opt.opts 

731 

732 def name(self) -> str: 

733 """Get the name that will be passed to the command function for this 

734 option. 

735 """ 

736 return cast(str, self._name) 

737 

738 def opts(self) -> list[str]: 

739 """Get the flags that will be used for this option on the command 

740 line. 

741 """ 

742 return self._opts 

743 

744 @property 

745 def help(self) -> str: 

746 """Get the help text for this option. Returns an empty string if no 

747 help was defined. 

748 """ 

749 return self.partialOpt.keywords.get("help", "") 

750 

751 def __call__(self, *args: Any, **kwargs: Any) -> Any: 

752 return self.partialOpt(*args, **kwargs) 

753 

754 

755class MWArgumentDecorator: 

756 """Wraps the click.argument decorator to enable shared arguments to be 

757 declared. 

758 """ 

759 

760 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

761 self._helpText = kwargs.pop("help", None) 

762 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

763 

764 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable: 

765 def decorator(f: Any) -> Any: 

766 if help is not None: 

767 self._helpText = help 

768 if self._helpText: 

769 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

770 return self.partialArg(*args, **kwargs)(f) 

771 

772 return decorator 

773 

774 

775class MWCommand(click.Command): 

776 """Command subclass that stores a copy of the args list for use by the 

777 command. 

778 """ 

779 

780 extra_epilog: str | None = None 

781 

782 def __init__(self, *args: Any, **kwargs: Any) -> None: 

783 # wrap callback method with catch_and_exit decorator 

784 callback = kwargs.get("callback") 

785 if callback is not None: 785 ↛ 788line 785 didn't jump to line 788, because the condition on line 785 was never false

786 kwargs = kwargs.copy() 

787 kwargs["callback"] = catch_and_exit(callback) 

788 super().__init__(*args, **kwargs) 

789 

790 def _capture_args(self, ctx: click.Context, args: list[str]) -> None: 

791 """Capture the command line options and arguments. 

792 

793 See details about what is captured and the order in which it is stored 

794 in the documentation of `MWCtxObj`. 

795 

796 Parameters 

797 ---------- 

798 ctx : `click.Context` 

799 The current Context. 

800 args : `list` [`str`] 

801 The list of arguments from the command line, split at spaces but 

802 not at separators (like "="). 

803 """ 

804 parser = self.make_parser(ctx) 

805 opts, _, param_order = parser.parse_args(args=list(args)) 

806 # `param_order` is a list of click.Option and click.Argument, there is 

807 # one item for each time the Option or Argument was used on the 

808 # command line. Options will precede Arguments, within each sublist 

809 # they are in the order they were used on the command line. Note that 

810 # click.Option and click.Argument do not contain the value from the 

811 # command line; values are in `opts`. 

812 # 

813 # `opts` is a dict where the key is the argument name to the 

814 # click.Command function, this name matches the `click.Option.name` or 

815 # `click.Argument.name`. For Options, an item will only be present if 

816 # the Option was used on the command line. For Arguments, an item will 

817 # always be present and if no value was provided on the command line 

818 # the value will be `None`. If the option accepts multiple values, the 

819 # value in `opts` is a tuple, otherwise it is a single item. 

820 next_idx: Counter = Counter() 

821 captured_args = [] 

822 for param in param_order: 

823 if isinstance(param, click.Option): 

824 param_name = cast(str, param.name) 

825 if param.multiple: 

826 val = opts[param_name][next_idx[param_name]] 

827 next_idx[param_name] += 1 

828 else: 

829 val = opts[param_name] 

830 if param.is_flag: 

831 # Bool options store their True flags in opts and their 

832 # False flags in secondary_opts. 

833 if val: 

834 flag = max(param.opts, key=len) 

835 else: 

836 flag = max(param.secondary_opts, key=len) 

837 captured_args.append(flag) 

838 else: 

839 captured_args.append(max(param.opts, key=len)) 

840 captured_args.append(val) 

841 elif isinstance(param, click.Argument): 

842 param_name = cast(str, param.name) 

843 if (opt := opts[param_name]) is not None: 

844 captured_args.append(opt) 

845 else: 

846 raise AssertionError("All parameters should be an Option or an Argument") 

847 MWCtxObj.getFrom(ctx).args = captured_args 

848 

849 def parse_args(self, ctx: click.Context, args: Any) -> list[str]: 

850 """Given a context and a list of arguments this creates the parser and 

851 parses the arguments, then modifies the context as necessary. This is 

852 automatically invoked by make_context(). 

853 

854 This function overrides `click.Command.parse_args`. 

855 

856 The call to `_capture_args` in this override stores the arguments 

857 (option names, option value, and argument values) that were used by the 

858 caller on the command line in the context object. These stored 

859 arugments can be used by the command function, e.g. to process options 

860 in the order they appeared on the command line (pipetask uses this 

861 feature to create pipeline actions in an order from different options). 

862 

863 Parameters 

864 ---------- 

865 ctx : `click.core.Context` 

866 The current Context.ß 

867 args : `list` [`str`] 

868 The list of arguments from the command line, split at spaces but 

869 not at separators (like "="). 

870 """ 

871 self._capture_args(ctx, args) 

872 return super().parse_args(ctx, args) 

873 

874 @property 

875 def epilog(self) -> str | None: 

876 """Override the epilog attribute to add extra_epilog (if defined by a 

877 subclass) to the end of any epilog provided by a subcommand. 

878 """ 

879 ret = self._epilog if self._epilog else "" 

880 if self.extra_epilog: 

881 if ret: 

882 ret += "\n\n" 

883 ret += self.extra_epilog 

884 return ret 

885 

886 @epilog.setter 

887 def epilog(self, val: str) -> None: 

888 self._epilog = val 

889 

890 

891class ButlerCommand(MWCommand): 

892 """Command subclass with butler-command specific overrides.""" 

893 

894 extra_epilog = "See 'butler --help' for more options." 

895 

896 

897class OptionGroup: 

898 """Base class for an option group decorator. Requires the option group 

899 subclass to have a property called `decorator`. 

900 """ 

901 

902 decorators: list[Any] 

903 

904 def __call__(self, f: Any) -> Any: 

905 for decorator in reversed(self.decorators): 

906 f = decorator(f) 

907 return f 

908 

909 

910class MWCtxObj: 

911 """Helper object for managing the `click.Context.obj` parameter, allows 

912 obj data to be managed in a consistent way. 

913 

914 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

915 initialize the obj if needed and return a new or existing `MWCtxObj`. 

916 

917 The `args` attribute contains a list of options, option values, and 

918 argument values that is similar to the list of arguments and options that 

919 were passed in on the command line, with differences noted below: 

920 

921 * Option namess and option values are first in the list, and argument 

922 values come last. The order of options and option values is preserved 

923 within the options. The order of argument values is preserved. 

924 

925 * The longest option name is used for the option in the `args` list, e.g. 

926 if an option accepts both short and long names "-o / --option" and the 

927 short option name "-o" was used on the command line, the longer name will 

928 be the one that appears in `args`. 

929 

930 * A long option name (which begins with two dashes "--") and its value may 

931 be separated by an equal sign; the name and value are split at the equal 

932 sign and it is removed. In `args`, the option is in one list item, and 

933 the option value (without the equal sign) is in the next list item. e.g. 

934 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

935 `args`. 

936 

937 * A short option name, (which begins with one dash "-") and its value are 

938 split immediately after the short option name, and if there is 

939 whitespace between the short option name and its value it is removed. 

940 Everything after the short option name (excluding whitespace) is included 

941 in the value. If the `Option` has a long name, the long name will be used 

942 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

943 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

944 `["--option", "=foo"]` (because everything after the short option name, 

945 except whitespace, is used for the value (as is standard with unix 

946 command line tools). 

947 

948 Attributes 

949 ---------- 

950 args : `list` [`str`] 

951 A list of options, option values, and arguments simialr to those that 

952 were passed in on the command line. See comments about captured options 

953 & arguments above. 

954 """ 

955 

956 def __init__(self) -> None: 

957 self.args = None 

958 

959 @staticmethod 

960 def getFrom(ctx: click.Context) -> Any: 

961 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

962 the new or already existing `MWCtxObj`. 

963 """ 

964 if ctx.obj is not None: 

965 return ctx.obj 

966 ctx.obj = MWCtxObj() 

967 return ctx.obj 

968 

969 

970def yaml_presets(ctx: click.Context, param: str, value: Any) -> None: 

971 """Read additional values from the supplied YAML file. 

972 

973 Parameters 

974 ---------- 

975 ctx : `click.context` 

976 The context for the click operation. Used to extract the subcommand 

977 name and translate option & argument names. 

978 param : `str` 

979 The parameter name. 

980 value : `object` 

981 The value of the parameter. 

982 """ 

983 

984 def _name_for_option(ctx: click.Context, option: str) -> str: 

985 """Use a CLI option name to find the name of the argument to the 

986 command function. 

987 

988 Parameters 

989 ---------- 

990 ctx : `click.Context` 

991 The context for the click operation. 

992 option : `str` 

993 The option/argument name from the yaml file. 

994 

995 Returns 

996 ------- 

997 name : str 

998 The name of the argument to use when calling the click.command 

999 function, as it should appear in the `ctx.default_map`. 

1000 

1001 Raises 

1002 ------ 

1003 RuntimeError 

1004 Raised if the option name from the yaml file does not exist in the 

1005 command parameters. This catches misspellings and incorrect useage 

1006 in the yaml file. 

1007 """ 

1008 for param in ctx.command.params: 

1009 # Remove leading dashes: they are not used for option names in the 

1010 # yaml file. 

1011 if option in [opt.lstrip("-") for opt in param.opts]: 

1012 return cast(str, param.name) 

1013 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

1014 

1015 ctx.default_map = ctx.default_map or {} 

1016 cmd_name = ctx.info_name 

1017 assert cmd_name is not None, "command name cannot be None" 

1018 if value: 

1019 try: 

1020 overrides = _read_yaml_presets(value, cmd_name) 

1021 options = list(overrides.keys()) 

1022 for option in options: 

1023 name = _name_for_option(ctx, option) 

1024 if name == option: 

1025 continue 

1026 overrides[name] = overrides.pop(option) 

1027 except Exception as e: 

1028 raise click.BadOptionUsage( 

1029 option_name=param, 

1030 message=f"Error reading overrides file: {e}", 

1031 ctx=ctx, 

1032 ) from None 

1033 # Override the defaults for this subcommand 

1034 ctx.default_map.update(overrides) 

1035 return 

1036 

1037 

1038def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]: 

1039 """Read file command line overrides from YAML config file. 

1040 

1041 Parameters 

1042 ---------- 

1043 file_uri : `str` 

1044 URI of override YAML file containing the command line overrides. 

1045 They should be grouped by command name. 

1046 cmd_name : `str` 

1047 The subcommand name that is being modified. 

1048 

1049 Returns 

1050 ------- 

1051 overrides : `dict` of [`str`, Any] 

1052 The relevant command line options read from the override file. 

1053 """ 

1054 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

1055 config = Config(file_uri) 

1056 return config[cmd_name] 

1057 

1058 

1059def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table: 

1060 """Sort an astropy table, with prioritization given to columns in this 

1061 order: 

1062 1. the provided named columns 

1063 2. spatial and temporal columns 

1064 3. the rest of the columns. 

1065 

1066 The table is sorted in-place, and is also returned for convenience. 

1067 

1068 Parameters 

1069 ---------- 

1070 table : `astropy.table.Table` 

1071 The table to sort 

1072 dimensions : `list` [``Dimension``] 

1073 The dimensions of the dataIds in the table (the dimensions should be 

1074 the same for all the dataIds). Used to determine if the column is 

1075 spatial, temporal, or neither. 

1076 sort_first : `list` [`str`] 

1077 The names of columns that should be sorted first, before spatial and 

1078 temporal columns. 

1079 

1080 Returns 

1081 ------- 

1082 `astropy.table.Table` 

1083 For convenience, the table that has been sorted. 

1084 """ 

1085 # For sorting we want to ignore the id 

1086 # We also want to move temporal or spatial dimensions earlier 

1087 sort_first = sort_first or [] 

1088 sort_early: list[str] = [] 

1089 sort_late: list[str] = [] 

1090 for dim in dimensions: 

1091 if dim.spatial or dim.temporal: 

1092 sort_early.extend(dim.required.names) 

1093 else: 

1094 sort_late.append(str(dim)) 

1095 sort_keys = sort_first + sort_early + sort_late 

1096 # The required names above means that we have the possibility of 

1097 # repeats of sort keys. Now have to remove them 

1098 # (order is retained by dict creation). 

1099 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1100 

1101 table.sort(sort_keys) 

1102 return table 

1103 

1104 

1105def catch_and_exit(func: Callable) -> Callable: 

1106 """Catch all exceptions, prints an exception traceback 

1107 and signals click to exit. 

1108 

1109 Use as decorator. 

1110 """ 

1111 

1112 @wraps(func) 

1113 def inner(*args: Any, **kwargs: Any) -> None: 

1114 try: 

1115 func(*args, **kwargs) 

1116 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1117 # this is handled by click itself 

1118 raise 

1119 except Exception: 

1120 exc_type, exc_value, exc_tb = sys.exc_info() 

1121 assert exc_type is not None 

1122 assert exc_value is not None 

1123 assert exc_tb is not None 

1124 if exc_tb.tb_next: 

1125 # do not show this decorator in traceback 

1126 exc_tb = exc_tb.tb_next 

1127 log.exception( 

1128 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1129 ) 

1130 # tell click to stop, this never returns. 

1131 click.get_current_context().exit(1) 

1132 

1133 return inner