Coverage for python/lsst/daf/butler/cli/utils.py: 34%

353 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 09:55 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21from __future__ import annotations 

22 

23__all__ = ( 

24 "astropyTablesToStr", 

25 "printAstropyTables", 

26 "textTypeStr", 

27 "LogCliRunner", 

28 "clickResultMsg", 

29 "command_test_env", 

30 "addArgumentHelp", 

31 "split_commas", 

32 "split_kv", 

33 "to_upper", 

34 "unwrap", 

35 "option_section", 

36 "MWPath", 

37 "MWOption", 

38 "MWArgument", 

39 "OptionSection", 

40 "MWOptionDecorator", 

41 "MWArgumentDecorator", 

42 "MWCommand", 

43 "ButlerCommand", 

44 "OptionGroup", 

45 "MWCtxObj", 

46 "yaml_presets", 

47 "sortAstropyTable", 

48 "catch_and_exit", 

49) 

50 

51 

52import itertools 

53import logging 

54import os 

55import re 

56import sys 

57import textwrap 

58import traceback 

59import uuid 

60import warnings 

61from collections import Counter 

62from collections.abc import Callable, Iterable, Iterator 

63from contextlib import contextmanager 

64from functools import partial, wraps 

65from typing import TYPE_CHECKING, Any, cast 

66from unittest.mock import patch 

67 

68import click 

69import click.exceptions 

70import click.testing 

71import yaml 

72from lsst.utils.iteration import ensure_iterable 

73 

74from ..core.config import Config 

75from .cliLog import CliLog 

76 

77if TYPE_CHECKING: 

78 from astropy.table import Table 

79 from lsst.daf.butler import Dimension 

80 

81log = logging.getLogger(__name__) 

82 

83# This is used as the metavar argument to Options that accept multiple string 

84# inputs, which may be comma-separarated. For example: 

85# --my-opt foo,bar --my-opt baz. 

86# Other arguments to the Option should include multiple=true and 

87# callback=split_kv. 

88typeStrAcceptsMultiple = "TEXT ..." 

89typeStrAcceptsSingle = "TEXT" 

90 

91# The standard help string for the --where option when it takes a WHERE clause. 

92where_help = ( 

93 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

94 "dimension table or a dimension name as a shortcut for the primary key column of a " 

95 "dimension table." 

96) 

97 

98 

99def astropyTablesToStr(tables: list[Table]) -> str: 

100 """Render astropy tables to string as they are displayed in the CLI. 

101 

102 Output formatting matches ``printAstropyTables``. 

103 """ 

104 ret = "" 

105 for table in tables: 

106 ret += "\n" 

107 table.pformat_all() 

108 ret += "\n" 

109 return ret 

110 

111 

112def printAstropyTables(tables: list[Table]) -> None: 

113 """Print astropy tables to be displayed in the CLI. 

114 

115 Output formatting matches ``astropyTablesToStr``. 

116 """ 

117 for table in tables: 

118 print("") 

119 table.pprint_all() 

120 print("") 

121 

122 

123def textTypeStr(multiple: bool) -> str: 

124 """Get the text type string for CLI help documentation. 

125 

126 Parameters 

127 ---------- 

128 multiple : `bool` 

129 True if multiple text values are allowed, False if only one value is 

130 allowed. 

131 

132 Returns 

133 ------- 

134 textTypeStr : `str` 

135 The type string to use. 

136 """ 

137 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

138 

139 

140class LogCliRunner(click.testing.CliRunner): 

141 """A test runner to use when the logging system will be initialized by code 

142 under test, calls CliLog.resetLog(), which undoes any logging setup that 

143 was done with the CliLog interface. 

144 

145 lsst.log modules can not be set back to an uninitialized state (python 

146 logging modules can be set back to NOTSET), instead they are set to 

147 `CliLog.defaultLsstLogLevel`. 

148 """ 

149 

150 def invoke(self, *args: Any, **kwargs: Any) -> Any: 

151 result = super().invoke(*args, **kwargs) 

152 CliLog.resetLog() 

153 return result 

154 

155 

156def clickResultMsg(result: click.testing.Result) -> str: 

157 """Get a standard assert message from a click result. 

158 

159 Parameters 

160 ---------- 

161 result : click.testing.Result 

162 The result object returned from click.testing.CliRunner.invoke 

163 

164 Returns 

165 ------- 

166 msg : `str` 

167 The message string. 

168 """ 

169 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

170 if result.exception: 

171 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

172 return msg 

173 

174 

175@contextmanager 

176def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]: 

177 """Context manager that creates (and then cleans up) an environment that 

178 provides a CLI plugin command with the given name. 

179 

180 Parameters 

181 ---------- 

182 runner : click.testing.CliRunner 

183 The test runner to use to create the isolated filesystem. 

184 commandModule : `str` 

185 The importable module that the command can be imported from. 

186 commandName : `str` 

187 The name of the command being published to import. 

188 """ 

189 with runner.isolated_filesystem(): 

190 with open("resources.yaml", "w") as f: 

191 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

192 # Add a colon to the end of the path on the next line, this tests the 

193 # case where the lookup in LoaderCLI._getPluginList generates an empty 

194 # string in one of the list entries and verifies that the empty string 

195 # is properly stripped out. 

196 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

197 yield 

198 

199 

200def addArgumentHelp(doc: str | None, helpText: str) -> str: 

201 """Add a Click argument's help message to a function's documentation. 

202 

203 This is needed because click presents arguments in the order the argument 

204 decorators are applied to a function, top down. But, the evaluation of the 

205 decorators happens bottom up, so if arguments just append their help to the 

206 function's docstring, the argument descriptions appear in reverse order 

207 from the order they are applied in. 

208 

209 Parameters 

210 ---------- 

211 doc : `str` 

212 The function's docstring. 

213 helpText : `str` 

214 The argument's help string to be inserted into the function's 

215 docstring. 

216 

217 Returns 

218 ------- 

219 doc : `str` 

220 Updated function documentation. 

221 """ 

222 if doc is None: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true

223 doc = helpText 

224 else: 

225 # See click documentation for details: 

226 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

227 # In short, text for the click command help can be truncated by putting 

228 # "\f" in the docstring, everything after it should be removed 

229 if "\f" in doc: 229 ↛ 230line 229 didn't jump to line 230, because the condition on line 229 was never true

230 doc = doc.split("\f")[0] 

231 

232 doclines = doc.splitlines() 

233 # The function's docstring may span multiple lines, so combine the 

234 # docstring from all the first lines until a blank line is encountered. 

235 # (Lines after the first blank line will be argument help.) 

236 while len(doclines) > 1 and doclines[1]: 

237 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

238 # Add standard indent to help text for proper alignment with command 

239 # function documentation: 

240 helpText = " " + helpText 

241 doclines.insert(1, helpText) 

242 doclines.insert(1, "\n") 

243 doc = "\n".join(doclines) 

244 return doc 

245 

246 

247def split_commas( 

248 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None 

249) -> tuple[str, ...]: 

250 """Process a tuple of values, where each value may contain comma-separated 

251 values, and return a single list of all the passed-in values. 

252 

253 This function can be passed to the 'callback' argument of a click.option to 

254 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If 

255 the comma is inside ``[]`` there will be no splitting. 

256 

257 Parameters 

258 ---------- 

259 context : `click.Context` or `None` 

260 The current execution context. Unused, but Click always passes it to 

261 callbacks. 

262 param : `click.core.Option` or `None` 

263 The parameter being handled. Unused, but Click always passes it to 

264 callbacks. 

265 values : iterable of `str` or `str` 

266 All the values passed for this option. Strings may contain commas, 

267 which will be treated as delimiters for separate values unless they 

268 are within ``[]``. 

269 

270 Returns 

271 ------- 

272 results : `tuple` [`str`] 

273 The passed in values separated by commas where appropriate and 

274 combined into a single tuple. 

275 """ 

276 if values is None: 

277 return tuple() 

278 valueList = [] 

279 for value in ensure_iterable(values): 

280 # If we have [, or ,] we do the slow split. If square brackets 

281 # are not matching then that is likely a typo that should result 

282 # in a warning. 

283 opens = "[" 

284 closes = "]" 

285 if re.search(rf"\{opens}.*,|,.*\{closes}", value): 

286 in_parens = False 

287 current = "" 

288 for c in value: 

289 if c == opens: 

290 if in_parens: 

291 warnings.warn( 

292 f"Found second opening {opens} without corresponding closing {closes}" 

293 f" in {value!r}", 

294 stacklevel=2, 

295 ) 

296 in_parens = True 

297 elif c == closes: 

298 if not in_parens: 

299 warnings.warn( 

300 f"Found second closing {closes} without corresponding open {opens} in {value!r}", 

301 stacklevel=2, 

302 ) 

303 in_parens = False 

304 elif c == ",": 

305 if not in_parens: 

306 # Split on this comma. 

307 valueList.append(current) 

308 current = "" 

309 continue 

310 current += c 

311 if in_parens: 

312 warnings.warn( 

313 f"Found opening {opens} that was never closed in {value!r}", 

314 stacklevel=2, 

315 ) 

316 if current: 

317 valueList.append(current) 

318 else: 

319 # Use efficient split since no parens. 

320 valueList.extend(value.split(",")) 

321 return tuple(valueList) 

322 

323 

324def split_kv( 

325 context: click.Context, 

326 param: click.core.Option, 

327 values: list[str], 

328 *, 

329 choice: click.Choice | None = None, 

330 multiple: bool = True, 

331 normalize: bool = False, 

332 separator: str = "=", 

333 unseparated_okay: bool = False, 

334 return_type: type[dict] | type[tuple] = dict, 

335 default_key: str = "", 

336 reverse_kv: bool = False, 

337 add_to_default: bool = False, 

338) -> dict[str, str] | tuple[tuple[str, str], ...]: 

339 """Process a tuple of values that are key-value pairs separated by a given 

340 separator. Multiple pairs may be comma separated. Return a dictionary of 

341 all the passed-in values. 

342 

343 This function can be passed to the 'callback' argument of a click.option to 

344 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

345 

346 Parameters 

347 ---------- 

348 context : `click.Context` or `None` 

349 The current execution context. Unused, but Click always passes it to 

350 callbacks. 

351 param : `click.core.Option` or `None` 

352 The parameter being handled. Unused, but Click always passes it to 

353 callbacks. 

354 values : [`str`] 

355 All the values passed for this option. Strings may contain commas, 

356 which will be treated as delimiters for separate values. 

357 choice : `click.Choice`, optional 

358 If provided, verify each value is a valid choice using the provided 

359 `click.Choice` instance. If None, no verification will be done. By 

360 default None 

361 multiple : `bool`, optional 

362 If true, the value may contain multiple comma-separated values. By 

363 default True. 

364 normalize : `bool`, optional 

365 If True and `choice.case_sensitive == False`, normalize the string the 

366 user provided to match the choice's case. By default False. 

367 separator : str, optional 

368 The character that separates key-value pairs. May not be a comma or an 

369 empty space (for space separators use Click's default implementation 

370 for tuples; `type=(str, str)`). By default "=". 

371 unseparated_okay : `bool`, optional 

372 If True, allow values that do not have a separator. They will be 

373 returned in the values dict as a tuple of values in the key '', that 

374 is: `values[''] = (unseparated_values, )`. By default False. 

375 return_type : `type`, must be `dict` or `tuple` 

376 The type of the value that should be returned. 

377 If `dict` then the returned object will be a dict, for each item in 

378 values, the value to the left of the separator will be the key and the 

379 value to the right of the separator will be the value. 

380 If `tuple` then the returned object will be a tuple. Each item in the 

381 tuple will be 2-item tuple, the first item will be the value to the 

382 left of the separator and the second item will be the value to the 

383 right. By default `dict`. 

384 default_key : `Any` 

385 The key to use if a value is passed that is not a key-value pair. 

386 (Passing values that are not key-value pairs requires 

387 ``unseparated_okay`` to be `True`.) 

388 reverse_kv : bool 

389 If true then for each item in values, the value to the left of the 

390 separator is treated as the value and the value to the right of the 

391 separator is treated as the key. By default False. 

392 add_to_default : `bool`, optional 

393 If True, then passed-in values will not overwrite the default value 

394 unless the ``return_type`` is `dict` and passed-in value(s) have the 

395 same key(s) as the default value. 

396 

397 Returns 

398 ------- 

399 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...] 

400 The passed-in values in dict form or tuple form. 

401 

402 Raises 

403 ------ 

404 `click.ClickException` 

405 Raised if the separator is not found in an entry, or if duplicate keys 

406 are encountered. 

407 """ 

408 

409 def norm(val: str) -> str: 

410 """If `normalize` is True and `choice` is not `None`, find the value 

411 in the available choices and return the value as spelled in the 

412 choices. 

413 

414 Assumes that val exists in choices; `split_kv` uses the `choice` 

415 instance to verify val is a valid choice. 

416 """ 

417 if normalize and choice is not None: 

418 v = val.casefold() 

419 for opt in choice.choices: 

420 if opt.casefold() == v: 

421 return opt 

422 return val 

423 

424 class RetDict: 

425 def __init__(self) -> None: 

426 self.ret: dict[str, str] = {} 

427 

428 def add(self, key: str, val: str) -> None: 

429 if reverse_kv: 

430 key, val = val, key 

431 self.ret[key] = val 

432 

433 def get(self) -> dict[str, str]: 

434 return self.ret 

435 

436 class RetTuple: 

437 def __init__(self) -> None: 

438 self.ret: list[tuple[str, str]] = [] 

439 

440 def add(self, key: str, val: str) -> None: 

441 if reverse_kv: 

442 key, val = val, key 

443 self.ret.append((key, val)) 

444 

445 def get(self) -> tuple[tuple[str, str], ...]: 

446 return tuple(self.ret) 

447 

448 if separator in (",", " "): 

449 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

450 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below. 

451 

452 if add_to_default: 

453 default = param.get_default(context) 

454 if default: 

455 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy 

456 

457 ret: RetDict | RetTuple 

458 if return_type is dict: 

459 ret = RetDict() 

460 elif return_type is tuple: 

461 ret = RetTuple() 

462 else: 

463 raise click.ClickException( 

464 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

465 ) 

466 if multiple: 

467 vals = split_commas(context, param, vals) 

468 for val in ensure_iterable(vals): 

469 if unseparated_okay and separator not in val: 

470 if choice is not None: 

471 choice(val) # will raise if val is an invalid choice 

472 ret.add(default_key, norm(val)) 

473 else: 

474 try: 

475 k, v = val.split(separator) 

476 if choice is not None: 

477 choice(v) # will raise if val is an invalid choice 

478 except ValueError as e: 

479 raise click.ClickException( 

480 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

481 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}" 

482 ) 

483 ret.add(k, norm(v)) 

484 return ret.get() 

485 

486 

487def to_upper(context: click.Context, param: click.core.Option, value: str) -> str: 

488 """Convert a value to upper case. 

489 

490 Parameters 

491 ---------- 

492 context : click.Context 

493 

494 values : string 

495 The value to be converted. 

496 

497 Returns 

498 ------- 

499 string 

500 A copy of the passed-in value, converted to upper case. 

501 """ 

502 return value.upper() 

503 

504 

505def unwrap(val: str) -> str: 

506 """Remove newlines and leading whitespace from a multi-line string with 

507 a consistent indentation level. 

508 

509 The first line of the string may be only a newline or may contain text 

510 followed by a newline, either is ok. After the first line, each line must 

511 begin with a consistant amount of whitespace. So, content of a 

512 triple-quoted string may begin immediately after the quotes, or the string 

513 may start with a newline. Each line after that must be the same amount of 

514 indentation/whitespace followed by text and a newline. The last line may 

515 end with a new line but is not required to do so. 

516 

517 Parameters 

518 ---------- 

519 val : `str` 

520 The string to change. 

521 

522 Returns 

523 ------- 

524 strippedString : `str` 

525 The string with newlines, indentation, and leading and trailing 

526 whitespace removed. 

527 """ 

528 

529 def splitSection(val: str) -> str: 

530 if not val.startswith("\n"): 530 ↛ 534line 530 didn't jump to line 534, because the condition on line 530 was never false

531 firstLine, _, val = val.partition("\n") 

532 firstLine += " " 

533 else: 

534 firstLine = "" 

535 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

536 

537 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

538 

539 

540class option_section: # noqa: N801 

541 """Decorator to add a section label between options in the help text of a 

542 command. 

543 

544 Parameters 

545 ---------- 

546 sectionText : `str` 

547 The text to print in the section identifier. 

548 """ 

549 

550 def __init__(self, sectionText: str) -> None: 

551 self.sectionText = "\n" + sectionText 

552 

553 def __call__(self, f: Any) -> click.Option: 

554 # Generate a parameter declaration that will be unique for this 

555 # section. 

556 return click.option( 

557 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

558 )(f) 

559 

560 

561class MWPath(click.Path): 

562 """Overrides click.Path to implement file-does-not-exist checking. 

563 

564 Changes the definition of ``exists` so that `True` indicates the location 

565 (file or directory) must exist, `False` indicates the location must *not* 

566 exist, and `None` indicates that the file may exist or not. The standard 

567 definition for the `click.Path` ``exists`` parameter is that for `True` a 

568 location must exist, but `False` means it is not required to exist (not 

569 that it is required to not exist). 

570 

571 Parameters 

572 ---------- 

573 exists : `True`, `False`, or `None` 

574 If `True`, the location (file or directory) indicated by the caller 

575 must exist. If `False` the location must not exist. If `None`, the 

576 location may exist or not. 

577 

578 For other parameters see `click.Path`. 

579 """ 

580 

581 def __init__( 

582 self, 

583 exists: bool | None = None, 

584 file_okay: bool = True, 

585 dir_okay: bool = True, 

586 writable: bool = False, 

587 readable: bool = True, 

588 resolve_path: bool = False, 

589 allow_dash: bool = False, 

590 path_type: type | None = None, 

591 ): 

592 self.mustNotExist = exists is False 

593 if exists is None: 593 ↛ 595line 593 didn't jump to line 595, because the condition on line 593 was never false

594 exists = False 

595 super().__init__( 

596 exists=exists, 

597 file_okay=file_okay, 

598 dir_okay=dir_okay, 

599 writable=writable, 

600 readable=readable, 

601 resolve_path=resolve_path, 

602 allow_dash=allow_dash, 

603 path_type=path_type, 

604 ) 

605 

606 def convert( 

607 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None 

608 ) -> Any: 

609 """Convert values through types. 

610 

611 Called by `click.ParamType` to "convert values through types". 

612 `click.Path` uses this step to verify Path conditions. 

613 """ 

614 if self.mustNotExist and os.path.exists(value): 

615 self.fail(f'Path "{value}" should not exist.') 

616 return super().convert(value, param, ctx) 

617 

618 

619class MWOption(click.Option): 

620 """Overrides click.Option with desired behaviors.""" 

621 

622 def make_metavar(self) -> str: 

623 """Make the metavar for the help menu. 

624 

625 Overrides `click.Option.make_metavar`. 

626 Adds a space and an ellipsis after the metavar name if 

627 the option accepts multiple inputs, otherwise defers to the base 

628 implementation. 

629 

630 By default click does not add an ellipsis when multiple is True and 

631 nargs is 1. And when nargs does not equal 1 click adds an ellipsis 

632 without a space between the metavar and the ellipsis, but we prefer a 

633 space between. 

634 

635 Does not get called for some option types (e.g. flag) so metavar 

636 transformation that must apply to all types should be applied in 

637 get_help_record. 

638 """ 

639 metavar = super().make_metavar() 

640 if self.multiple and self.nargs == 1: 

641 metavar += " ..." 

642 elif self.nargs != 1: 

643 metavar = f"{metavar[:-3]} ..." 

644 return metavar 

645 

646 

647class MWArgument(click.Argument): 

648 """Overrides click.Argument with desired behaviors.""" 

649 

650 def make_metavar(self) -> str: 

651 """Make the metavar for the help menu. 

652 

653 Overrides `click.Option.make_metavar`. 

654 Always adds a space and an ellipsis (' ...') after the 

655 metavar name if the option accepts multiple inputs. 

656 

657 By default click adds an ellipsis without a space between the metavar 

658 and the ellipsis, but we prefer a space between. 

659 

660 Returns 

661 ------- 

662 metavar : `str` 

663 The metavar value. 

664 """ 

665 metavar = super().make_metavar() 

666 if self.nargs != 1: 

667 metavar = f"{metavar[:-3]} ..." 

668 return metavar 

669 

670 

671class OptionSection(MWOption): 

672 """Implements an Option that prints a section label in the help text and 

673 does not pass any value to the command function. 

674 

675 This class does a bit of hackery to add a section label to a click command 

676 help output: first, `expose_value` is set to `False` so that no value is 

677 passed to the command function. Second, this class overrides 

678 `click.Option.get_help_record` to return the section label string without 

679 any prefix so that it stands out as a section label. 

680 

681 This class overrides the hidden attribute because our documentation build 

682 tool, sphinx-click, implements its own `get_help_record` function which 

683 builds the record from other option values (e.g. `name`, `opts`), which 

684 breaks the hack we use to make `get_help_record` only return the 

685 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

686 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

687 entering its `_get_help_record` function. So, making the hidden property 

688 return True hides this option from sphinx-click, while allowing the section 

689 text to be returned by our `get_help_record` method when using Click. 

690 

691 The intention for this implementation is to do minimally invasive overrides 

692 of the click classes so as to be robust and easy to fix if the click 

693 internals change. 

694 

695 Parameters 

696 ---------- 

697 sectionName : `str` 

698 The parameter declaration for this option. It is not shown to the user, 

699 it must be unique within the command. If using the `section` decorator 

700 to add a section to a command's options, the section name is 

701 auto-generated. 

702 sectionText : `str` 

703 The text to print in the section identifier. 

704 """ 

705 

706 @property 

707 def hidden(self) -> bool: 

708 return True 

709 

710 @hidden.setter 

711 def hidden(self, val: Any) -> None: 

712 pass 

713 

714 def __init__(self, sectionName: str, sectionText: str) -> None: 

715 super().__init__(sectionName, expose_value=False) 

716 self.sectionText = sectionText 

717 

718 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]: 

719 return (self.sectionText, "") 

720 

721 

722class MWOptionDecorator: 

723 """Wraps the click.option decorator to enable shared options to be declared 

724 and allows inspection of the shared option. 

725 """ 

726 

727 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

728 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

729 opt = click.Option(param_decls, **kwargs) 

730 self._name = opt.name 

731 self._opts = opt.opts 

732 

733 def name(self) -> str: 

734 """Get the name that will be passed to the command function for this 

735 option. 

736 """ 

737 return cast(str, self._name) 

738 

739 def opts(self) -> list[str]: 

740 """Get the flags that will be used for this option on the command 

741 line. 

742 """ 

743 return self._opts 

744 

745 @property 

746 def help(self) -> str: 

747 """Get the help text for this option. Returns an empty string if no 

748 help was defined. 

749 """ 

750 return self.partialOpt.keywords.get("help", "") 

751 

752 def __call__(self, *args: Any, **kwargs: Any) -> Any: 

753 return self.partialOpt(*args, **kwargs) 

754 

755 

756class MWArgumentDecorator: 

757 """Wraps the click.argument decorator to enable shared arguments to be 

758 declared. 

759 """ 

760 

761 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

762 self._helpText = kwargs.pop("help", None) 

763 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

764 

765 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable: 

766 def decorator(f: Any) -> Any: 

767 if help is not None: 

768 self._helpText = help 

769 if self._helpText: 

770 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

771 return self.partialArg(*args, **kwargs)(f) 

772 

773 return decorator 

774 

775 

776class MWCommand(click.Command): 

777 """Command subclass that stores a copy of the args list for use by the 

778 command. 

779 """ 

780 

781 extra_epilog: str | None = None 

782 

783 def __init__(self, *args: Any, **kwargs: Any) -> None: 

784 # wrap callback method with catch_and_exit decorator 

785 callback = kwargs.get("callback") 

786 if callback is not None: 786 ↛ 789line 786 didn't jump to line 789, because the condition on line 786 was never false

787 kwargs = kwargs.copy() 

788 kwargs["callback"] = catch_and_exit(callback) 

789 super().__init__(*args, **kwargs) 

790 

791 def _capture_args(self, ctx: click.Context, args: list[str]) -> None: 

792 """Capture the command line options and arguments. 

793 

794 See details about what is captured and the order in which it is stored 

795 in the documentation of `MWCtxObj`. 

796 

797 Parameters 

798 ---------- 

799 ctx : `click.Context` 

800 The current Context. 

801 args : `list` [`str`] 

802 The list of arguments from the command line, split at spaces but 

803 not at separators (like "="). 

804 """ 

805 parser = self.make_parser(ctx) 

806 opts, _, param_order = parser.parse_args(args=list(args)) 

807 # `param_order` is a list of click.Option and click.Argument, there is 

808 # one item for each time the Option or Argument was used on the 

809 # command line. Options will precede Arguments, within each sublist 

810 # they are in the order they were used on the command line. Note that 

811 # click.Option and click.Argument do not contain the value from the 

812 # command line; values are in `opts`. 

813 # 

814 # `opts` is a dict where the key is the argument name to the 

815 # click.Command function, this name matches the `click.Option.name` or 

816 # `click.Argument.name`. For Options, an item will only be present if 

817 # the Option was used on the command line. For Arguments, an item will 

818 # always be present and if no value was provided on the command line 

819 # the value will be `None`. If the option accepts multiple values, the 

820 # value in `opts` is a tuple, otherwise it is a single item. 

821 next_idx: Counter = Counter() 

822 captured_args = [] 

823 for param in param_order: 

824 if isinstance(param, click.Option): 

825 param_name = cast(str, param.name) 

826 if param.multiple: 

827 val = opts[param_name][next_idx[param_name]] 

828 next_idx[param_name] += 1 

829 else: 

830 val = opts[param_name] 

831 if param.is_flag: 

832 # Bool options store their True flags in opts and their 

833 # False flags in secondary_opts. 

834 if val: 

835 flag = max(param.opts, key=len) 

836 else: 

837 flag = max(param.secondary_opts, key=len) 

838 captured_args.append(flag) 

839 else: 

840 captured_args.append(max(param.opts, key=len)) 

841 captured_args.append(val) 

842 elif isinstance(param, click.Argument): 

843 param_name = cast(str, param.name) 

844 if (opt := opts[param_name]) is not None: 

845 captured_args.append(opt) 

846 else: 

847 assert False # All parameters should be an Option or an Argument. 

848 MWCtxObj.getFrom(ctx).args = captured_args 

849 

850 def parse_args(self, ctx: click.Context, args: Any) -> list[str]: 

851 """Given a context and a list of arguments this creates the parser and 

852 parses the arguments, then modifies the context as necessary. This is 

853 automatically invoked by make_context(). 

854 

855 This function overrides `click.Command.parse_args`. 

856 

857 The call to `_capture_args` in this override stores the arguments 

858 (option names, option value, and argument values) that were used by the 

859 caller on the command line in the context object. These stored 

860 arugments can be used by the command function, e.g. to process options 

861 in the order they appeared on the command line (pipetask uses this 

862 feature to create pipeline actions in an order from different options). 

863 

864 Parameters 

865 ---------- 

866 ctx : `click.core.Context` 

867 The current Context.ß 

868 args : `list` [`str`] 

869 The list of arguments from the command line, split at spaces but 

870 not at separators (like "="). 

871 """ 

872 self._capture_args(ctx, args) 

873 return super().parse_args(ctx, args) 

874 

875 @property 

876 def epilog(self) -> str | None: 

877 """Override the epilog attribute to add extra_epilog (if defined by a 

878 subclass) to the end of any epilog provided by a subcommand. 

879 """ 

880 ret = self._epilog if self._epilog else "" 

881 if self.extra_epilog: 

882 if ret: 

883 ret += "\n\n" 

884 ret += self.extra_epilog 

885 return ret 

886 

887 @epilog.setter 

888 def epilog(self, val: str) -> None: 

889 self._epilog = val 

890 

891 

892class ButlerCommand(MWCommand): 

893 """Command subclass with butler-command specific overrides.""" 

894 

895 extra_epilog = "See 'butler --help' for more options." 

896 

897 

898class OptionGroup: 

899 """Base class for an option group decorator. Requires the option group 

900 subclass to have a property called `decorator`. 

901 """ 

902 

903 decorators: list[Any] 

904 

905 def __call__(self, f: Any) -> Any: 

906 for decorator in reversed(self.decorators): 

907 f = decorator(f) 

908 return f 

909 

910 

911class MWCtxObj: 

912 """Helper object for managing the `click.Context.obj` parameter, allows 

913 obj data to be managed in a consistent way. 

914 

915 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

916 initialize the obj if needed and return a new or existing `MWCtxObj`. 

917 

918 The `args` attribute contains a list of options, option values, and 

919 argument values that is similar to the list of arguments and options that 

920 were passed in on the command line, with differences noted below: 

921 

922 * Option namess and option values are first in the list, and argument 

923 values come last. The order of options and option values is preserved 

924 within the options. The order of argument values is preserved. 

925 

926 * The longest option name is used for the option in the `args` list, e.g. 

927 if an option accepts both short and long names "-o / --option" and the 

928 short option name "-o" was used on the command line, the longer name will 

929 be the one that appears in `args`. 

930 

931 * A long option name (which begins with two dashes "--") and its value may 

932 be separated by an equal sign; the name and value are split at the equal 

933 sign and it is removed. In `args`, the option is in one list item, and 

934 the option value (without the equal sign) is in the next list item. e.g. 

935 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

936 `args`. 

937 

938 * A short option name, (which begins with one dash "-") and its value are 

939 split immediately after the short option name, and if there is 

940 whitespace between the short option name and its value it is removed. 

941 Everything after the short option name (excluding whitespace) is included 

942 in the value. If the `Option` has a long name, the long name will be used 

943 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

944 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

945 `["--option", "=foo"]` (because everything after the short option name, 

946 except whitespace, is used for the value (as is standard with unix 

947 command line tools). 

948 

949 Attributes 

950 ---------- 

951 args : `list` [`str`] 

952 A list of options, option values, and arguments simialr to those that 

953 were passed in on the command line. See comments about captured options 

954 & arguments above. 

955 """ 

956 

957 def __init__(self) -> None: 

958 self.args = None 

959 

960 @staticmethod 

961 def getFrom(ctx: click.Context) -> Any: 

962 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

963 the new or already existing `MWCtxObj`. 

964 """ 

965 if ctx.obj is not None: 

966 return ctx.obj 

967 ctx.obj = MWCtxObj() 

968 return ctx.obj 

969 

970 

971def yaml_presets(ctx: click.Context, param: str, value: Any) -> None: 

972 """Read additional values from the supplied YAML file. 

973 

974 Parameters 

975 ---------- 

976 ctx : `click.context` 

977 The context for the click operation. Used to extract the subcommand 

978 name and translate option & argument names. 

979 param : `str` 

980 The parameter name. 

981 value : `object` 

982 The value of the parameter. 

983 """ 

984 

985 def _name_for_option(ctx: click.Context, option: str) -> str: 

986 """Use a CLI option name to find the name of the argument to the 

987 command function. 

988 

989 Parameters 

990 ---------- 

991 ctx : `click.Context` 

992 The context for the click operation. 

993 option : `str` 

994 The option/argument name from the yaml file. 

995 

996 Returns 

997 ------- 

998 name : str 

999 The name of the argument to use when calling the click.command 

1000 function, as it should appear in the `ctx.default_map`. 

1001 

1002 Raises 

1003 ------ 

1004 RuntimeError 

1005 Raised if the option name from the yaml file does not exist in the 

1006 command parameters. This catches misspellings and incorrect useage 

1007 in the yaml file. 

1008 """ 

1009 for param in ctx.command.params: 

1010 # Remove leading dashes: they are not used for option names in the 

1011 # yaml file. 

1012 if option in [opt.lstrip("-") for opt in param.opts]: 

1013 return cast(str, param.name) 

1014 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

1015 

1016 ctx.default_map = ctx.default_map or {} 

1017 cmd_name = ctx.info_name 

1018 assert cmd_name is not None, "command name cannot be None" 

1019 if value: 

1020 try: 

1021 overrides = _read_yaml_presets(value, cmd_name) 

1022 options = list(overrides.keys()) 

1023 for option in options: 

1024 name = _name_for_option(ctx, option) 

1025 if name == option: 

1026 continue 

1027 overrides[name] = overrides.pop(option) 

1028 except Exception as e: 

1029 raise click.BadOptionUsage( 

1030 option_name=param, 

1031 message=f"Error reading overrides file: {e}", 

1032 ctx=ctx, 

1033 ) 

1034 # Override the defaults for this subcommand 

1035 ctx.default_map.update(overrides) 

1036 return 

1037 

1038 

1039def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]: 

1040 """Read file command line overrides from YAML config file. 

1041 

1042 Parameters 

1043 ---------- 

1044 file_uri : `str` 

1045 URI of override YAML file containing the command line overrides. 

1046 They should be grouped by command name. 

1047 cmd_name : `str` 

1048 The subcommand name that is being modified. 

1049 

1050 Returns 

1051 ------- 

1052 overrides : `dict` of [`str`, Any] 

1053 The relevant command line options read from the override file. 

1054 """ 

1055 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

1056 config = Config(file_uri) 

1057 return config[cmd_name] 

1058 

1059 

1060def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table: 

1061 """Sort an astropy table, with prioritization given to columns in this 

1062 order: 

1063 1. the provided named columns 

1064 2. spatial and temporal columns 

1065 3. the rest of the columns. 

1066 

1067 The table is sorted in-place, and is also returned for convenience. 

1068 

1069 Parameters 

1070 ---------- 

1071 table : `astropy.table.Table` 

1072 The table to sort 

1073 dimensions : `list` [``Dimension``] 

1074 The dimensions of the dataIds in the table (the dimensions should be 

1075 the same for all the dataIds). Used to determine if the column is 

1076 spatial, temporal, or neither. 

1077 sort_first : `list` [`str`] 

1078 The names of columns that should be sorted first, before spatial and 

1079 temporal columns. 

1080 

1081 Returns 

1082 ------- 

1083 `astropy.table.Table` 

1084 For convenience, the table that has been sorted. 

1085 """ 

1086 # For sorting we want to ignore the id 

1087 # We also want to move temporal or spatial dimensions earlier 

1088 sort_first = sort_first or [] 

1089 sort_early: list[str] = [] 

1090 sort_late: list[str] = [] 

1091 for dim in dimensions: 

1092 if dim.spatial or dim.temporal: 

1093 sort_early.extend(dim.required.names) 

1094 else: 

1095 sort_late.append(str(dim)) 

1096 sort_keys = sort_first + sort_early + sort_late 

1097 # The required names above means that we have the possibility of 

1098 # repeats of sort keys. Now have to remove them 

1099 # (order is retained by dict creation). 

1100 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1101 

1102 table.sort(sort_keys) 

1103 return table 

1104 

1105 

1106def catch_and_exit(func: Callable) -> Callable: 

1107 """Catch all exceptions, prints an exception traceback 

1108 and signals click to exit. 

1109 

1110 Use as decorator. 

1111 """ 

1112 

1113 @wraps(func) 

1114 def inner(*args: Any, **kwargs: Any) -> None: 

1115 try: 

1116 func(*args, **kwargs) 

1117 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1118 # this is handled by click itself 

1119 raise 

1120 except Exception: 

1121 exc_type, exc_value, exc_tb = sys.exc_info() 

1122 assert exc_type is not None 

1123 assert exc_value is not None 

1124 assert exc_tb is not None 

1125 if exc_tb.tb_next: 

1126 # do not show this decorator in traceback 

1127 exc_tb = exc_tb.tb_next 

1128 log.exception( 

1129 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1130 ) 

1131 # tell click to stop, this never returns. 

1132 click.get_current_context().exit(1) 

1133 

1134 return inner