Coverage for python/lsst/daf/butler/cli/utils.py: 35%

351 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-12-05 11:07 +0000

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27from __future__ import annotations 

28 

29__all__ = ( 

30 "astropyTablesToStr", 

31 "printAstropyTables", 

32 "textTypeStr", 

33 "LogCliRunner", 

34 "clickResultMsg", 

35 "command_test_env", 

36 "addArgumentHelp", 

37 "split_commas", 

38 "split_kv", 

39 "to_upper", 

40 "unwrap", 

41 "option_section", 

42 "MWPath", 

43 "MWOption", 

44 "MWArgument", 

45 "OptionSection", 

46 "MWOptionDecorator", 

47 "MWArgumentDecorator", 

48 "MWCommand", 

49 "ButlerCommand", 

50 "OptionGroup", 

51 "MWCtxObj", 

52 "yaml_presets", 

53 "sortAstropyTable", 

54 "catch_and_exit", 

55) 

56 

57 

58import itertools 

59import logging 

60import os 

61import re 

62import sys 

63import textwrap 

64import traceback 

65import uuid 

66import warnings 

67from collections import Counter 

68from collections.abc import Callable, Iterable, Iterator 

69from contextlib import contextmanager 

70from functools import partial, wraps 

71from typing import TYPE_CHECKING, Any, cast 

72from unittest.mock import patch 

73 

74import click 

75import click.exceptions 

76import click.testing 

77import yaml 

78from lsst.utils.iteration import ensure_iterable 

79 

80from .._config import Config 

81from .cliLog import CliLog 

82 

83if TYPE_CHECKING: 

84 from astropy.table import Table 

85 from lsst.daf.butler import Dimension 

86 

87log = logging.getLogger(__name__) 

88 

89# This is used as the metavar argument to Options that accept multiple string 

90# inputs, which may be comma-separarated. For example: 

91# --my-opt foo,bar --my-opt baz. 

92# Other arguments to the Option should include multiple=true and 

93# callback=split_kv. 

94typeStrAcceptsMultiple = "TEXT ..." 

95typeStrAcceptsSingle = "TEXT" 

96 

97# The standard help string for the --where option when it takes a WHERE clause. 

98where_help = ( 

99 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

100 "dimension table or a dimension name as a shortcut for the primary key column of a " 

101 "dimension table." 

102) 

103 

104 

105def astropyTablesToStr(tables: list[Table]) -> str: 

106 """Render astropy tables to string as they are displayed in the CLI. 

107 

108 Output formatting matches ``printAstropyTables``. 

109 """ 

110 ret = "" 

111 for table in tables: 

112 ret += "\n" 

113 table.pformat_all() 

114 ret += "\n" 

115 return ret 

116 

117 

118def printAstropyTables(tables: list[Table]) -> None: 

119 """Print astropy tables to be displayed in the CLI. 

120 

121 Output formatting matches ``astropyTablesToStr``. 

122 """ 

123 for table in tables: 

124 print("") 

125 table.pprint_all() 

126 print("") 

127 

128 

129def textTypeStr(multiple: bool) -> str: 

130 """Get the text type string for CLI help documentation. 

131 

132 Parameters 

133 ---------- 

134 multiple : `bool` 

135 True if multiple text values are allowed, False if only one value is 

136 allowed. 

137 

138 Returns 

139 ------- 

140 textTypeStr : `str` 

141 The type string to use. 

142 """ 

143 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

144 

145 

146class LogCliRunner(click.testing.CliRunner): 

147 """A test runner to use when the logging system will be initialized by code 

148 under test, calls CliLog.resetLog(), which undoes any logging setup that 

149 was done with the CliLog interface. 

150 

151 lsst.log modules can not be set back to an uninitialized state (python 

152 logging modules can be set back to NOTSET), instead they are set to 

153 `CliLog.defaultLsstLogLevel`. 

154 """ 

155 

156 def invoke(self, *args: Any, **kwargs: Any) -> Any: 

157 result = super().invoke(*args, **kwargs) 

158 CliLog.resetLog() 

159 return result 

160 

161 

162def clickResultMsg(result: click.testing.Result) -> str: 

163 """Get a standard assert message from a click result. 

164 

165 Parameters 

166 ---------- 

167 result : click.testing.Result 

168 The result object returned from click.testing.CliRunner.invoke 

169 

170 Returns 

171 ------- 

172 msg : `str` 

173 The message string. 

174 """ 

175 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

176 if result.exception: 

177 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

178 return msg 

179 

180 

181@contextmanager 

182def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]: 

183 """Context manager that creates (and then cleans up) an environment that 

184 provides a CLI plugin command with the given name. 

185 

186 Parameters 

187 ---------- 

188 runner : click.testing.CliRunner 

189 The test runner to use to create the isolated filesystem. 

190 commandModule : `str` 

191 The importable module that the command can be imported from. 

192 commandName : `str` 

193 The name of the command being published to import. 

194 """ 

195 with runner.isolated_filesystem(): 

196 with open("resources.yaml", "w") as f: 

197 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

198 # Add a colon to the end of the path on the next line, this tests the 

199 # case where the lookup in LoaderCLI._getPluginList generates an empty 

200 # string in one of the list entries and verifies that the empty string 

201 # is properly stripped out. 

202 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

203 yield 

204 

205 

206def addArgumentHelp(doc: str | None, helpText: str) -> str: 

207 """Add a Click argument's help message to a function's documentation. 

208 

209 This is needed because click presents arguments in the order the argument 

210 decorators are applied to a function, top down. But, the evaluation of the 

211 decorators happens bottom up, so if arguments just append their help to the 

212 function's docstring, the argument descriptions appear in reverse order 

213 from the order they are applied in. 

214 

215 Parameters 

216 ---------- 

217 doc : `str` 

218 The function's docstring. 

219 helpText : `str` 

220 The argument's help string to be inserted into the function's 

221 docstring. 

222 

223 Returns 

224 ------- 

225 doc : `str` 

226 Updated function documentation. 

227 """ 

228 if doc is None: 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true

229 doc = helpText 

230 else: 

231 # See click documentation for details: 

232 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

233 # In short, text for the click command help can be truncated by putting 

234 # "\f" in the docstring, everything after it should be removed 

235 if "\f" in doc: 235 ↛ 236line 235 didn't jump to line 236, because the condition on line 235 was never true

236 doc = doc.split("\f")[0] 

237 

238 doclines = doc.splitlines() 

239 # The function's docstring may span multiple lines, so combine the 

240 # docstring from all the first lines until a blank line is encountered. 

241 # (Lines after the first blank line will be argument help.) 

242 while len(doclines) > 1 and doclines[1]: 

243 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

244 # Add standard indent to help text for proper alignment with command 

245 # function documentation: 

246 helpText = " " + helpText 

247 doclines.insert(1, helpText) 

248 doclines.insert(1, "\n") 

249 doc = "\n".join(doclines) 

250 return doc 

251 

252 

253def split_commas( 

254 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None 

255) -> tuple[str, ...]: 

256 """Process a tuple of values, where each value may contain comma-separated 

257 values, and return a single list of all the passed-in values. 

258 

259 This function can be passed to the 'callback' argument of a click.option to 

260 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If 

261 the comma is inside ``[]`` there will be no splitting. 

262 

263 Parameters 

264 ---------- 

265 context : `click.Context` or `None` 

266 The current execution context. Unused, but Click always passes it to 

267 callbacks. 

268 param : `click.core.Option` or `None` 

269 The parameter being handled. Unused, but Click always passes it to 

270 callbacks. 

271 values : iterable of `str` or `str` 

272 All the values passed for this option. Strings may contain commas, 

273 which will be treated as delimiters for separate values unless they 

274 are within ``[]``. 

275 

276 Returns 

277 ------- 

278 results : `tuple` [`str`] 

279 The passed in values separated by commas where appropriate and 

280 combined into a single tuple. 

281 """ 

282 if values is None: 

283 return () 

284 valueList = [] 

285 for value in ensure_iterable(values): 

286 # If we have [, or ,] we do the slow split. If square brackets 

287 # are not matching then that is likely a typo that should result 

288 # in a warning. 

289 opens = "[" 

290 closes = "]" 

291 if re.search(rf"\{opens}.*,|,.*\{closes}", value): 

292 in_parens = False 

293 current = "" 

294 for c in value: 

295 if c == opens: 

296 if in_parens: 

297 warnings.warn( 

298 f"Found second opening {opens} without corresponding closing {closes}" 

299 f" in {value!r}", 

300 stacklevel=2, 

301 ) 

302 in_parens = True 

303 elif c == closes: 

304 if not in_parens: 

305 warnings.warn( 

306 f"Found second closing {closes} without corresponding open {opens} in {value!r}", 

307 stacklevel=2, 

308 ) 

309 in_parens = False 

310 elif c == "," and not in_parens: 

311 # Split on this comma. 

312 valueList.append(current) 

313 current = "" 

314 continue 

315 current += c 

316 if in_parens: 

317 warnings.warn( 

318 f"Found opening {opens} that was never closed in {value!r}", 

319 stacklevel=2, 

320 ) 

321 if current: 

322 valueList.append(current) 

323 else: 

324 # Use efficient split since no parens. 

325 valueList.extend(value.split(",")) 

326 return tuple(valueList) 

327 

328 

329def split_kv( 

330 context: click.Context, 

331 param: click.core.Option, 

332 values: list[str], 

333 *, 

334 choice: click.Choice | None = None, 

335 multiple: bool = True, 

336 normalize: bool = False, 

337 separator: str = "=", 

338 unseparated_okay: bool = False, 

339 return_type: type[dict] | type[tuple] = dict, 

340 default_key: str = "", 

341 reverse_kv: bool = False, 

342 add_to_default: bool = False, 

343) -> dict[str, str] | tuple[tuple[str, str], ...]: 

344 """Process a tuple of values that are key-value pairs separated by a given 

345 separator. Multiple pairs may be comma separated. Return a dictionary of 

346 all the passed-in values. 

347 

348 This function can be passed to the 'callback' argument of a click.option to 

349 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

350 

351 Parameters 

352 ---------- 

353 context : `click.Context` or `None` 

354 The current execution context. Unused, but Click always passes it to 

355 callbacks. 

356 param : `click.core.Option` or `None` 

357 The parameter being handled. Unused, but Click always passes it to 

358 callbacks. 

359 values : [`str`] 

360 All the values passed for this option. Strings may contain commas, 

361 which will be treated as delimiters for separate values. 

362 choice : `click.Choice`, optional 

363 If provided, verify each value is a valid choice using the provided 

364 `click.Choice` instance. If None, no verification will be done. By 

365 default None 

366 multiple : `bool`, optional 

367 If true, the value may contain multiple comma-separated values. By 

368 default True. 

369 normalize : `bool`, optional 

370 If True and `choice.case_sensitive == False`, normalize the string the 

371 user provided to match the choice's case. By default False. 

372 separator : str, optional 

373 The character that separates key-value pairs. May not be a comma or an 

374 empty space (for space separators use Click's default implementation 

375 for tuples; `type=(str, str)`). By default "=". 

376 unseparated_okay : `bool`, optional 

377 If True, allow values that do not have a separator. They will be 

378 returned in the values dict as a tuple of values in the key '', that 

379 is: `values[''] = (unseparated_values, )`. By default False. 

380 return_type : `type`, must be `dict` or `tuple` 

381 The type of the value that should be returned. 

382 If `dict` then the returned object will be a dict, for each item in 

383 values, the value to the left of the separator will be the key and the 

384 value to the right of the separator will be the value. 

385 If `tuple` then the returned object will be a tuple. Each item in the 

386 tuple will be 2-item tuple, the first item will be the value to the 

387 left of the separator and the second item will be the value to the 

388 right. By default `dict`. 

389 default_key : `Any` 

390 The key to use if a value is passed that is not a key-value pair. 

391 (Passing values that are not key-value pairs requires 

392 ``unseparated_okay`` to be `True`.) 

393 reverse_kv : bool 

394 If true then for each item in values, the value to the left of the 

395 separator is treated as the value and the value to the right of the 

396 separator is treated as the key. By default False. 

397 add_to_default : `bool`, optional 

398 If True, then passed-in values will not overwrite the default value 

399 unless the ``return_type`` is `dict` and passed-in value(s) have the 

400 same key(s) as the default value. 

401 

402 Returns 

403 ------- 

404 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...] 

405 The passed-in values in dict form or tuple form. 

406 

407 Raises 

408 ------ 

409 `click.ClickException` 

410 Raised if the separator is not found in an entry, or if duplicate keys 

411 are encountered. 

412 """ 

413 

414 def norm(val: str) -> str: 

415 """If `normalize` is True and `choice` is not `None`, find the value 

416 in the available choices and return the value as spelled in the 

417 choices. 

418 

419 Assumes that val exists in choices; `split_kv` uses the `choice` 

420 instance to verify val is a valid choice. 

421 """ 

422 if normalize and choice is not None: 

423 v = val.casefold() 

424 for opt in choice.choices: 

425 if opt.casefold() == v: 

426 return opt 

427 return val 

428 

429 class RetDict: 

430 def __init__(self) -> None: 

431 self.ret: dict[str, str] = {} 

432 

433 def add(self, key: str, val: str) -> None: 

434 if reverse_kv: 

435 key, val = val, key 

436 self.ret[key] = val 

437 

438 def get(self) -> dict[str, str]: 

439 return self.ret 

440 

441 class RetTuple: 

442 def __init__(self) -> None: 

443 self.ret: list[tuple[str, str]] = [] 

444 

445 def add(self, key: str, val: str) -> None: 

446 if reverse_kv: 

447 key, val = val, key 

448 self.ret.append((key, val)) 

449 

450 def get(self) -> tuple[tuple[str, str], ...]: 

451 return tuple(self.ret) 

452 

453 if separator in (",", " "): 

454 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

455 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below. 

456 

457 if add_to_default: 

458 default = param.get_default(context) 

459 if default: 

460 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy 

461 

462 ret: RetDict | RetTuple 

463 if return_type is dict: 

464 ret = RetDict() 

465 elif return_type is tuple: 

466 ret = RetTuple() 

467 else: 

468 raise click.ClickException( 

469 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

470 ) 

471 if multiple: 

472 vals = split_commas(context, param, vals) 

473 for val in ensure_iterable(vals): 

474 if unseparated_okay and separator not in val: 

475 if choice is not None: 

476 choice(val) # will raise if val is an invalid choice 

477 ret.add(default_key, norm(val)) 

478 else: 

479 try: 

480 k, v = val.split(separator) 

481 if choice is not None: 

482 choice(v) # will raise if val is an invalid choice 

483 except ValueError as e: 

484 raise click.ClickException( 

485 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

486 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}" 

487 ) from None 

488 ret.add(k, norm(v)) 

489 return ret.get() 

490 

491 

492def to_upper(context: click.Context, param: click.core.Option, value: str) -> str: 

493 """Convert a value to upper case. 

494 

495 Parameters 

496 ---------- 

497 context : click.Context 

498 

499 values : string 

500 The value to be converted. 

501 

502 Returns 

503 ------- 

504 string 

505 A copy of the passed-in value, converted to upper case. 

506 """ 

507 return value.upper() 

508 

509 

510def unwrap(val: str) -> str: 

511 """Remove newlines and leading whitespace from a multi-line string with 

512 a consistent indentation level. 

513 

514 The first line of the string may be only a newline or may contain text 

515 followed by a newline, either is ok. After the first line, each line must 

516 begin with a consistant amount of whitespace. So, content of a 

517 triple-quoted string may begin immediately after the quotes, or the string 

518 may start with a newline. Each line after that must be the same amount of 

519 indentation/whitespace followed by text and a newline. The last line may 

520 end with a new line but is not required to do so. 

521 

522 Parameters 

523 ---------- 

524 val : `str` 

525 The string to change. 

526 

527 Returns 

528 ------- 

529 strippedString : `str` 

530 The string with newlines, indentation, and leading and trailing 

531 whitespace removed. 

532 """ 

533 

534 def splitSection(val: str) -> str: 

535 if not val.startswith("\n"): 535 ↛ 539line 535 didn't jump to line 539, because the condition on line 535 was never false

536 firstLine, _, val = val.partition("\n") 

537 firstLine += " " 

538 else: 

539 firstLine = "" 

540 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

541 

542 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

543 

544 

545class option_section: # noqa: N801 

546 """Decorator to add a section label between options in the help text of a 

547 command. 

548 

549 Parameters 

550 ---------- 

551 sectionText : `str` 

552 The text to print in the section identifier. 

553 """ 

554 

555 def __init__(self, sectionText: str) -> None: 

556 self.sectionText = "\n" + sectionText 

557 

558 def __call__(self, f: Any) -> click.Option: 

559 # Generate a parameter declaration that will be unique for this 

560 # section. 

561 return click.option( 

562 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

563 )(f) 

564 

565 

566class MWPath(click.Path): 

567 """Overrides click.Path to implement file-does-not-exist checking. 

568 

569 Changes the definition of ``exists` so that `True` indicates the location 

570 (file or directory) must exist, `False` indicates the location must *not* 

571 exist, and `None` indicates that the file may exist or not. The standard 

572 definition for the `click.Path` ``exists`` parameter is that for `True` a 

573 location must exist, but `False` means it is not required to exist (not 

574 that it is required to not exist). 

575 

576 Parameters 

577 ---------- 

578 exists : `True`, `False`, or `None` 

579 If `True`, the location (file or directory) indicated by the caller 

580 must exist. If `False` the location must not exist. If `None`, the 

581 location may exist or not. 

582 

583 For other parameters see `click.Path`. 

584 """ 

585 

586 def __init__( 

587 self, 

588 exists: bool | None = None, 

589 file_okay: bool = True, 

590 dir_okay: bool = True, 

591 writable: bool = False, 

592 readable: bool = True, 

593 resolve_path: bool = False, 

594 allow_dash: bool = False, 

595 path_type: type | None = None, 

596 ): 

597 self.mustNotExist = exists is False 

598 if exists is None: 598 ↛ 600line 598 didn't jump to line 600, because the condition on line 598 was never false

599 exists = False 

600 super().__init__( 

601 exists=exists, 

602 file_okay=file_okay, 

603 dir_okay=dir_okay, 

604 writable=writable, 

605 readable=readable, 

606 resolve_path=resolve_path, 

607 allow_dash=allow_dash, 

608 path_type=path_type, 

609 ) 

610 

611 def convert( 

612 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None 

613 ) -> Any: 

614 """Convert values through types. 

615 

616 Called by `click.ParamType` to "convert values through types". 

617 `click.Path` uses this step to verify Path conditions. 

618 """ 

619 if self.mustNotExist and os.path.exists(value): 

620 self.fail(f'Path "{value}" should not exist.') 

621 return super().convert(value, param, ctx) 

622 

623 

624class MWOption(click.Option): 

625 """Overrides click.Option with desired behaviors.""" 

626 

627 def make_metavar(self) -> str: 

628 """Make the metavar for the help menu. 

629 

630 Overrides `click.Option.make_metavar`. 

631 Adds a space and an ellipsis after the metavar name if 

632 the option accepts multiple inputs, otherwise defers to the base 

633 implementation. 

634 

635 By default click does not add an ellipsis when multiple is True and 

636 nargs is 1. And when nargs does not equal 1 click adds an ellipsis 

637 without a space between the metavar and the ellipsis, but we prefer a 

638 space between. 

639 

640 Does not get called for some option types (e.g. flag) so metavar 

641 transformation that must apply to all types should be applied in 

642 get_help_record. 

643 """ 

644 metavar = super().make_metavar() 

645 if self.multiple and self.nargs == 1: 

646 metavar += " ..." 

647 elif self.nargs != 1: 

648 metavar = f"{metavar[:-3]} ..." 

649 return metavar 

650 

651 

652class MWArgument(click.Argument): 

653 """Overrides click.Argument with desired behaviors.""" 

654 

655 def make_metavar(self) -> str: 

656 """Make the metavar for the help menu. 

657 

658 Overrides `click.Option.make_metavar`. 

659 Always adds a space and an ellipsis (' ...') after the 

660 metavar name if the option accepts multiple inputs. 

661 

662 By default click adds an ellipsis without a space between the metavar 

663 and the ellipsis, but we prefer a space between. 

664 

665 Returns 

666 ------- 

667 metavar : `str` 

668 The metavar value. 

669 """ 

670 metavar = super().make_metavar() 

671 if self.nargs != 1: 

672 metavar = f"{metavar[:-3]} ..." 

673 return metavar 

674 

675 

676class OptionSection(MWOption): 

677 """Implements an Option that prints a section label in the help text and 

678 does not pass any value to the command function. 

679 

680 This class does a bit of hackery to add a section label to a click command 

681 help output: first, `expose_value` is set to `False` so that no value is 

682 passed to the command function. Second, this class overrides 

683 `click.Option.get_help_record` to return the section label string without 

684 any prefix so that it stands out as a section label. 

685 

686 This class overrides the hidden attribute because our documentation build 

687 tool, sphinx-click, implements its own `get_help_record` function which 

688 builds the record from other option values (e.g. `name`, `opts`), which 

689 breaks the hack we use to make `get_help_record` only return the 

690 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

691 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

692 entering its `_get_help_record` function. So, making the hidden property 

693 return True hides this option from sphinx-click, while allowing the section 

694 text to be returned by our `get_help_record` method when using Click. 

695 

696 The intention for this implementation is to do minimally invasive overrides 

697 of the click classes so as to be robust and easy to fix if the click 

698 internals change. 

699 

700 Parameters 

701 ---------- 

702 sectionName : `str` 

703 The parameter declaration for this option. It is not shown to the user, 

704 it must be unique within the command. If using the `section` decorator 

705 to add a section to a command's options, the section name is 

706 auto-generated. 

707 sectionText : `str` 

708 The text to print in the section identifier. 

709 """ 

710 

711 @property 

712 def hidden(self) -> bool: 

713 return True 

714 

715 @hidden.setter 

716 def hidden(self, val: Any) -> None: 

717 pass 

718 

719 def __init__(self, sectionName: str, sectionText: str) -> None: 

720 super().__init__(sectionName, expose_value=False) 

721 self.sectionText = sectionText 

722 

723 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]: 

724 return (self.sectionText, "") 

725 

726 

727class MWOptionDecorator: 

728 """Wraps the click.option decorator to enable shared options to be declared 

729 and allows inspection of the shared option. 

730 """ 

731 

732 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

733 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

734 opt = click.Option(param_decls, **kwargs) 

735 self._name = opt.name 

736 self._opts = opt.opts 

737 

738 def name(self) -> str: 

739 """Get the name that will be passed to the command function for this 

740 option. 

741 """ 

742 return cast(str, self._name) 

743 

744 def opts(self) -> list[str]: 

745 """Get the flags that will be used for this option on the command 

746 line. 

747 """ 

748 return self._opts 

749 

750 @property 

751 def help(self) -> str: 

752 """Get the help text for this option. Returns an empty string if no 

753 help was defined. 

754 """ 

755 return self.partialOpt.keywords.get("help", "") 

756 

757 def __call__(self, *args: Any, **kwargs: Any) -> Any: 

758 return self.partialOpt(*args, **kwargs) 

759 

760 

761class MWArgumentDecorator: 

762 """Wraps the click.argument decorator to enable shared arguments to be 

763 declared. 

764 """ 

765 

766 def __init__(self, *param_decls: Any, **kwargs: Any) -> None: 

767 self._helpText = kwargs.pop("help", None) 

768 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

769 

770 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable: 

771 def decorator(f: Any) -> Any: 

772 if help is not None: 

773 self._helpText = help 

774 if self._helpText: 

775 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

776 return self.partialArg(*args, **kwargs)(f) 

777 

778 return decorator 

779 

780 

781class MWCommand(click.Command): 

782 """Command subclass that stores a copy of the args list for use by the 

783 command. 

784 """ 

785 

786 extra_epilog: str | None = None 

787 

788 def __init__(self, *args: Any, **kwargs: Any) -> None: 

789 # wrap callback method with catch_and_exit decorator 

790 callback = kwargs.get("callback") 

791 if callback is not None: 791 ↛ 794line 791 didn't jump to line 794, because the condition on line 791 was never false

792 kwargs = kwargs.copy() 

793 kwargs["callback"] = catch_and_exit(callback) 

794 super().__init__(*args, **kwargs) 

795 

796 def _capture_args(self, ctx: click.Context, args: list[str]) -> None: 

797 """Capture the command line options and arguments. 

798 

799 See details about what is captured and the order in which it is stored 

800 in the documentation of `MWCtxObj`. 

801 

802 Parameters 

803 ---------- 

804 ctx : `click.Context` 

805 The current Context. 

806 args : `list` [`str`] 

807 The list of arguments from the command line, split at spaces but 

808 not at separators (like "="). 

809 """ 

810 parser = self.make_parser(ctx) 

811 opts, _, param_order = parser.parse_args(args=list(args)) 

812 # `param_order` is a list of click.Option and click.Argument, there is 

813 # one item for each time the Option or Argument was used on the 

814 # command line. Options will precede Arguments, within each sublist 

815 # they are in the order they were used on the command line. Note that 

816 # click.Option and click.Argument do not contain the value from the 

817 # command line; values are in `opts`. 

818 # 

819 # `opts` is a dict where the key is the argument name to the 

820 # click.Command function, this name matches the `click.Option.name` or 

821 # `click.Argument.name`. For Options, an item will only be present if 

822 # the Option was used on the command line. For Arguments, an item will 

823 # always be present and if no value was provided on the command line 

824 # the value will be `None`. If the option accepts multiple values, the 

825 # value in `opts` is a tuple, otherwise it is a single item. 

826 next_idx: Counter = Counter() 

827 captured_args = [] 

828 for param in param_order: 

829 if isinstance(param, click.Option): 

830 param_name = cast(str, param.name) 

831 if param.multiple: 

832 val = opts[param_name][next_idx[param_name]] 

833 next_idx[param_name] += 1 

834 else: 

835 val = opts[param_name] 

836 if param.is_flag: 

837 # Bool options store their True flags in opts and their 

838 # False flags in secondary_opts. 

839 if val: 

840 flag = max(param.opts, key=len) 

841 else: 

842 flag = max(param.secondary_opts, key=len) 

843 captured_args.append(flag) 

844 else: 

845 captured_args.append(max(param.opts, key=len)) 

846 captured_args.append(val) 

847 elif isinstance(param, click.Argument): 

848 param_name = cast(str, param.name) 

849 if (opt := opts[param_name]) is not None: 

850 captured_args.append(opt) 

851 else: 

852 raise AssertionError("All parameters should be an Option or an Argument") 

853 MWCtxObj.getFrom(ctx).args = captured_args 

854 

855 def parse_args(self, ctx: click.Context, args: Any) -> list[str]: 

856 """Given a context and a list of arguments this creates the parser and 

857 parses the arguments, then modifies the context as necessary. This is 

858 automatically invoked by make_context(). 

859 

860 This function overrides `click.Command.parse_args`. 

861 

862 The call to `_capture_args` in this override stores the arguments 

863 (option names, option value, and argument values) that were used by the 

864 caller on the command line in the context object. These stored 

865 arugments can be used by the command function, e.g. to process options 

866 in the order they appeared on the command line (pipetask uses this 

867 feature to create pipeline actions in an order from different options). 

868 

869 Parameters 

870 ---------- 

871 ctx : `click.core.Context` 

872 The current Context.ß 

873 args : `list` [`str`] 

874 The list of arguments from the command line, split at spaces but 

875 not at separators (like "="). 

876 """ 

877 self._capture_args(ctx, args) 

878 return super().parse_args(ctx, args) 

879 

880 @property 

881 def epilog(self) -> str | None: 

882 """Override the epilog attribute to add extra_epilog (if defined by a 

883 subclass) to the end of any epilog provided by a subcommand. 

884 """ 

885 ret = self._epilog if self._epilog else "" 

886 if self.extra_epilog: 

887 if ret: 

888 ret += "\n\n" 

889 ret += self.extra_epilog 

890 return ret 

891 

892 @epilog.setter 

893 def epilog(self, val: str) -> None: 

894 self._epilog = val 

895 

896 

897class ButlerCommand(MWCommand): 

898 """Command subclass with butler-command specific overrides.""" 

899 

900 extra_epilog = "See 'butler --help' for more options." 

901 

902 

903class OptionGroup: 

904 """Base class for an option group decorator. Requires the option group 

905 subclass to have a property called `decorator`. 

906 """ 

907 

908 decorators: list[Any] 

909 

910 def __call__(self, f: Any) -> Any: 

911 for decorator in reversed(self.decorators): 

912 f = decorator(f) 

913 return f 

914 

915 

916class MWCtxObj: 

917 """Helper object for managing the `click.Context.obj` parameter, allows 

918 obj data to be managed in a consistent way. 

919 

920 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

921 initialize the obj if needed and return a new or existing `MWCtxObj`. 

922 

923 The `args` attribute contains a list of options, option values, and 

924 argument values that is similar to the list of arguments and options that 

925 were passed in on the command line, with differences noted below: 

926 

927 * Option namess and option values are first in the list, and argument 

928 values come last. The order of options and option values is preserved 

929 within the options. The order of argument values is preserved. 

930 

931 * The longest option name is used for the option in the `args` list, e.g. 

932 if an option accepts both short and long names "-o / --option" and the 

933 short option name "-o" was used on the command line, the longer name will 

934 be the one that appears in `args`. 

935 

936 * A long option name (which begins with two dashes "--") and its value may 

937 be separated by an equal sign; the name and value are split at the equal 

938 sign and it is removed. In `args`, the option is in one list item, and 

939 the option value (without the equal sign) is in the next list item. e.g. 

940 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

941 `args`. 

942 

943 * A short option name, (which begins with one dash "-") and its value are 

944 split immediately after the short option name, and if there is 

945 whitespace between the short option name and its value it is removed. 

946 Everything after the short option name (excluding whitespace) is included 

947 in the value. If the `Option` has a long name, the long name will be used 

948 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

949 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

950 `["--option", "=foo"]` (because everything after the short option name, 

951 except whitespace, is used for the value (as is standard with unix 

952 command line tools). 

953 

954 Attributes 

955 ---------- 

956 args : `list` [`str`] 

957 A list of options, option values, and arguments simialr to those that 

958 were passed in on the command line. See comments about captured options 

959 & arguments above. 

960 """ 

961 

962 def __init__(self) -> None: 

963 self.args = None 

964 

965 @staticmethod 

966 def getFrom(ctx: click.Context) -> Any: 

967 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

968 the new or already existing `MWCtxObj`. 

969 """ 

970 if ctx.obj is not None: 

971 return ctx.obj 

972 ctx.obj = MWCtxObj() 

973 return ctx.obj 

974 

975 

976def yaml_presets(ctx: click.Context, param: str, value: Any) -> None: 

977 """Read additional values from the supplied YAML file. 

978 

979 Parameters 

980 ---------- 

981 ctx : `click.context` 

982 The context for the click operation. Used to extract the subcommand 

983 name and translate option & argument names. 

984 param : `str` 

985 The parameter name. 

986 value : `object` 

987 The value of the parameter. 

988 """ 

989 

990 def _name_for_option(ctx: click.Context, option: str) -> str: 

991 """Use a CLI option name to find the name of the argument to the 

992 command function. 

993 

994 Parameters 

995 ---------- 

996 ctx : `click.Context` 

997 The context for the click operation. 

998 option : `str` 

999 The option/argument name from the yaml file. 

1000 

1001 Returns 

1002 ------- 

1003 name : str 

1004 The name of the argument to use when calling the click.command 

1005 function, as it should appear in the `ctx.default_map`. 

1006 

1007 Raises 

1008 ------ 

1009 RuntimeError 

1010 Raised if the option name from the yaml file does not exist in the 

1011 command parameters. This catches misspellings and incorrect useage 

1012 in the yaml file. 

1013 """ 

1014 for param in ctx.command.params: 

1015 # Remove leading dashes: they are not used for option names in the 

1016 # yaml file. 

1017 if option in [opt.lstrip("-") for opt in param.opts]: 

1018 return cast(str, param.name) 

1019 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

1020 

1021 ctx.default_map = ctx.default_map or {} 

1022 cmd_name = ctx.info_name 

1023 assert cmd_name is not None, "command name cannot be None" 

1024 if value: 

1025 try: 

1026 overrides = _read_yaml_presets(value, cmd_name) 

1027 options = list(overrides.keys()) 

1028 for option in options: 

1029 name = _name_for_option(ctx, option) 

1030 if name == option: 

1031 continue 

1032 overrides[name] = overrides.pop(option) 

1033 except Exception as e: 

1034 raise click.BadOptionUsage( 

1035 option_name=param, 

1036 message=f"Error reading overrides file: {e}", 

1037 ctx=ctx, 

1038 ) from None 

1039 # Override the defaults for this subcommand 

1040 ctx.default_map.update(overrides) 

1041 return 

1042 

1043 

1044def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]: 

1045 """Read file command line overrides from YAML config file. 

1046 

1047 Parameters 

1048 ---------- 

1049 file_uri : `str` 

1050 URI of override YAML file containing the command line overrides. 

1051 They should be grouped by command name. 

1052 cmd_name : `str` 

1053 The subcommand name that is being modified. 

1054 

1055 Returns 

1056 ------- 

1057 overrides : `dict` of [`str`, Any] 

1058 The relevant command line options read from the override file. 

1059 """ 

1060 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

1061 config = Config(file_uri) 

1062 return config[cmd_name] 

1063 

1064 

1065def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table: 

1066 """Sort an astropy table, with prioritization given to columns in this 

1067 order: 

1068 1. the provided named columns 

1069 2. spatial and temporal columns 

1070 3. the rest of the columns. 

1071 

1072 The table is sorted in-place, and is also returned for convenience. 

1073 

1074 Parameters 

1075 ---------- 

1076 table : `astropy.table.Table` 

1077 The table to sort 

1078 dimensions : `list` [``Dimension``] 

1079 The dimensions of the dataIds in the table (the dimensions should be 

1080 the same for all the dataIds). Used to determine if the column is 

1081 spatial, temporal, or neither. 

1082 sort_first : `list` [`str`] 

1083 The names of columns that should be sorted first, before spatial and 

1084 temporal columns. 

1085 

1086 Returns 

1087 ------- 

1088 `astropy.table.Table` 

1089 For convenience, the table that has been sorted. 

1090 """ 

1091 # For sorting we want to ignore the id 

1092 # We also want to move temporal or spatial dimensions earlier 

1093 sort_first = sort_first or [] 

1094 sort_early: list[str] = [] 

1095 sort_late: list[str] = [] 

1096 for dim in dimensions: 

1097 if dim.spatial or dim.temporal: 

1098 sort_early.extend(dim.required.names) 

1099 else: 

1100 sort_late.append(str(dim)) 

1101 sort_keys = sort_first + sort_early + sort_late 

1102 # The required names above means that we have the possibility of 

1103 # repeats of sort keys. Now have to remove them 

1104 # (order is retained by dict creation). 

1105 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1106 

1107 table.sort(sort_keys) 

1108 return table 

1109 

1110 

1111def catch_and_exit(func: Callable) -> Callable: 

1112 """Catch all exceptions, prints an exception traceback 

1113 and signals click to exit. 

1114 

1115 Use as decorator. 

1116 """ 

1117 

1118 @wraps(func) 

1119 def inner(*args: Any, **kwargs: Any) -> None: 

1120 try: 

1121 func(*args, **kwargs) 

1122 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1123 # this is handled by click itself 

1124 raise 

1125 except Exception: 

1126 exc_type, exc_value, exc_tb = sys.exc_info() 

1127 assert exc_type is not None 

1128 assert exc_value is not None 

1129 assert exc_tb is not None 

1130 if exc_tb.tb_next: 

1131 # do not show this decorator in traceback 

1132 exc_tb = exc_tb.tb_next 

1133 log.exception( 

1134 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1135 ) 

1136 # tell click to stop, this never returns. 

1137 click.get_current_context().exit(1) 

1138 

1139 return inner