Coverage for python/lsst/daf/butler/cli/utils.py: 32%

343 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2022-11-15 01:59 -0800

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22 

23__all__ = ( 

24 "astropyTablesToStr", 

25 "printAstropyTables", 

26 "textTypeStr", 

27 "LogCliRunner", 

28 "clickResultMsg", 

29 "command_test_env", 

30 "addArgumentHelp", 

31 "split_commas", 

32 "split_kv", 

33 "to_upper", 

34 "unwrap", 

35 "option_section", 

36 "MWPath", 

37 "MWOption", 

38 "MWArgument", 

39 "OptionSection", 

40 "MWOptionDecorator", 

41 "MWArgumentDecorator", 

42 "MWCommand", 

43 "ButlerCommand", 

44 "OptionGroup", 

45 "MWCtxObj", 

46 "yaml_presets", 

47 "sortAstropyTable", 

48 "catch_and_exit", 

49) 

50 

51 

52import itertools 

53import logging 

54import os 

55import re 

56import sys 

57import textwrap 

58import traceback 

59import uuid 

60import warnings 

61from collections import Counter 

62from contextlib import contextmanager 

63from functools import partial, wraps 

64from unittest.mock import patch 

65 

66import click 

67import click.exceptions 

68import click.testing 

69import yaml 

70from lsst.utils.iteration import ensure_iterable 

71 

72from ..core.config import Config 

73from .cliLog import CliLog 

74 

75log = logging.getLogger(__name__) 

76 

77# This is used as the metavar argument to Options that accept multiple string 

78# inputs, which may be comma-separarated. For example: 

79# --my-opt foo,bar --my-opt baz. 

80# Other arguments to the Option should include multiple=true and 

81# callback=split_kv. 

82typeStrAcceptsMultiple = "TEXT ..." 

83typeStrAcceptsSingle = "TEXT" 

84 

85# The standard help string for the --where option when it takes a WHERE clause. 

86where_help = ( 

87 "A string expression similar to a SQL WHERE clause. May involve any column of a " 

88 "dimension table or a dimension name as a shortcut for the primary key column of a " 

89 "dimension table." 

90) 

91 

92 

93def astropyTablesToStr(tables): 

94 """Render astropy tables to string as they are displayed in the CLI. 

95 

96 Output formatting matches ``printAstropyTables``. 

97 """ 

98 ret = "" 

99 for table in tables: 

100 ret += "\n" 

101 table.pformat_all() 

102 ret += "\n" 

103 return ret 

104 

105 

106def printAstropyTables(tables): 

107 """Print astropy tables to be displayed in the CLI. 

108 

109 Output formatting matches ``astropyTablesToStr``. 

110 """ 

111 for table in tables: 

112 print("") 

113 table.pprint_all() 

114 print("") 

115 

116 

117def textTypeStr(multiple): 

118 """Get the text type string for CLI help documentation. 

119 

120 Parameters 

121 ---------- 

122 multiple : `bool` 

123 True if multiple text values are allowed, False if only one value is 

124 allowed. 

125 

126 Returns 

127 ------- 

128 textTypeStr : `str` 

129 The type string to use. 

130 """ 

131 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle 

132 

133 

134class LogCliRunner(click.testing.CliRunner): 

135 """A test runner to use when the logging system will be initialized by code 

136 under test, calls CliLog.resetLog(), which undoes any logging setup that 

137 was done with the CliLog interface. 

138 

139 lsst.log modules can not be set back to an uninitialized state (python 

140 logging modules can be set back to NOTSET), instead they are set to 

141 `CliLog.defaultLsstLogLevel`.""" 

142 

143 def invoke(self, *args, **kwargs): 

144 result = super().invoke(*args, **kwargs) 

145 CliLog.resetLog() 

146 return result 

147 

148 

149def clickResultMsg(result): 

150 """Get a standard assert message from a click result 

151 

152 Parameters 

153 ---------- 

154 result : click.Result 

155 The result object returned from click.testing.CliRunner.invoke 

156 

157 Returns 

158 ------- 

159 msg : `str` 

160 The message string. 

161 """ 

162 msg = f"""\noutput: {result.output}\nexception: {result.exception}""" 

163 if result.exception: 

164 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}""" 

165 return msg 

166 

167 

168@contextmanager 

169def command_test_env(runner, commandModule, commandName): 

170 """A context manager that creates (and then cleans up) an environment that 

171 provides a CLI plugin command with the given name. 

172 

173 Parameters 

174 ---------- 

175 runner : click.testing.CliRunner 

176 The test runner to use to create the isolated filesystem. 

177 commandModule : `str` 

178 The importable module that the command can be imported from. 

179 commandName : `str` 

180 The name of the command being published to import. 

181 """ 

182 with runner.isolated_filesystem(): 

183 with open("resources.yaml", "w") as f: 

184 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}})) 

185 # Add a colon to the end of the path on the next line, this tests the 

186 # case where the lookup in LoaderCLI._getPluginList generates an empty 

187 # string in one of the list entries and verifies that the empty string 

188 # is properly stripped out. 

189 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}): 

190 yield 

191 

192 

193def addArgumentHelp(doc, helpText): 

194 """Add a Click argument's help message to a function's documentation. 

195 

196 This is needed because click presents arguments in the order the argument 

197 decorators are applied to a function, top down. But, the evaluation of the 

198 decorators happens bottom up, so if arguments just append their help to the 

199 function's docstring, the argument descriptions appear in reverse order 

200 from the order they are applied in. 

201 

202 Parameters 

203 ---------- 

204 doc : `str` 

205 The function's docstring. 

206 helpText : `str` 

207 The argument's help string to be inserted into the function's 

208 docstring. 

209 

210 Returns 

211 ------- 

212 doc : `str` 

213 Updated function documentation. 

214 """ 

215 if doc is None: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true

216 doc = helpText 

217 else: 

218 # See click documentation for details: 

219 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts 

220 # In short, text for the click command help can be truncated by putting 

221 # "\f" in the docstring, everything after it should be removed 

222 if "\f" in doc: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true

223 doc = doc.split("\f")[0] 

224 

225 doclines = doc.splitlines() 

226 # The function's docstring may span multiple lines, so combine the 

227 # docstring from all the first lines until a blank line is encountered. 

228 # (Lines after the first blank line will be argument help.) 

229 while len(doclines) > 1 and doclines[1]: 

230 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip())) 

231 # Add standard indent to help text for proper alignment with command 

232 # function documentation: 

233 helpText = " " + helpText 

234 doclines.insert(1, helpText) 

235 doclines.insert(1, "\n") 

236 doc = "\n".join(doclines) 

237 return doc 

238 

239 

240def split_commas(context, param, values): 

241 """Process a tuple of values, where each value may contain comma-separated 

242 values, and return a single list of all the passed-in values. 

243 

244 This function can be passed to the 'callback' argument of a click.option to 

245 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If 

246 the comma is inside ``[]`` there will be no splitting. 

247 

248 Parameters 

249 ---------- 

250 context : `click.Context` or `None` 

251 The current execution context. Unused, but Click always passes it to 

252 callbacks. 

253 param : `click.core.Option` or `None` 

254 The parameter being handled. Unused, but Click always passes it to 

255 callbacks. 

256 values : iterable of `str` or `str` 

257 All the values passed for this option. Strings may contain commas, 

258 which will be treated as delimiters for separate values unless they 

259 are within ``[]``. 

260 

261 Returns 

262 ------- 

263 results : `tuple` [`str`] 

264 The passed in values separated by commas where appropriate and 

265 combined into a single tuple. 

266 """ 

267 if values is None: 

268 return values 

269 valueList = [] 

270 for value in ensure_iterable(values): 

271 # If we have [, or ,] we do the slow split. If square brackets 

272 # are not matching then that is likely a typo that should result 

273 # in a warning. 

274 opens = "[" 

275 closes = "]" 

276 if re.search(rf"\{opens}.*,|,.*\{closes}", value): 

277 in_parens = False 

278 current = "" 

279 for c in value: 

280 if c == opens: 

281 if in_parens: 

282 warnings.warn( 

283 f"Found second opening {opens} without corresponding closing {closes}" 

284 f" in {value!r}", 

285 stacklevel=2, 

286 ) 

287 in_parens = True 

288 elif c == closes: 

289 if not in_parens: 

290 warnings.warn( 

291 f"Found second closing {closes} without corresponding open {opens} in {value!r}", 

292 stacklevel=2, 

293 ) 

294 in_parens = False 

295 elif c == ",": 

296 if not in_parens: 

297 # Split on this comma. 

298 valueList.append(current) 

299 current = "" 

300 continue 

301 current += c 

302 if in_parens: 

303 warnings.warn( 

304 f"Found opening {opens} that was never closed in {value!r}", 

305 stacklevel=2, 

306 ) 

307 if current: 

308 valueList.append(current) 

309 else: 

310 # Use efficient split since no parens. 

311 valueList.extend(value.split(",")) 

312 return tuple(valueList) 

313 

314 

315def split_kv( 

316 context, 

317 param, 

318 values, 

319 choice=None, 

320 multiple=True, 

321 normalize=False, 

322 separator="=", 

323 unseparated_okay=False, 

324 return_type=dict, 

325 default_key="", 

326 reverse_kv=False, 

327 add_to_default=False, 

328): 

329 """Process a tuple of values that are key-value pairs separated by a given 

330 separator. Multiple pairs may be comma separated. Return a dictionary of 

331 all the passed-in values. 

332 

333 This function can be passed to the 'callback' argument of a click.option to 

334 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2"). 

335 

336 Parameters 

337 ---------- 

338 context : `click.Context` or `None` 

339 The current execution context. Unused, but Click always passes it to 

340 callbacks. 

341 param : `click.core.Option` or `None` 

342 The parameter being handled. Unused, but Click always passes it to 

343 callbacks. 

344 values : [`str`] 

345 All the values passed for this option. Strings may contain commas, 

346 which will be treated as delimiters for separate values. 

347 choice : `click.Choice`, optional 

348 If provided, verify each value is a valid choice using the provided 

349 `click.Choice` instance. If None, no verification will be done. By 

350 default None 

351 multiple : `bool`, optional 

352 If true, the value may contain multiple comma-separated values. By 

353 default True. 

354 normalize : `bool`, optional 

355 If True and `choice.case_sensitive == False`, normalize the string the 

356 user provided to match the choice's case. By default False. 

357 separator : str, optional 

358 The character that separates key-value pairs. May not be a comma or an 

359 empty space (for space separators use Click's default implementation 

360 for tuples; `type=(str, str)`). By default "=". 

361 unseparated_okay : `bool`, optional 

362 If True, allow values that do not have a separator. They will be 

363 returned in the values dict as a tuple of values in the key '', that 

364 is: `values[''] = (unseparated_values, )`. By default False. 

365 return_type : `type`, must be `dict` or `tuple` 

366 The type of the value that should be returned. 

367 If `dict` then the returned object will be a dict, for each item in 

368 values, the value to the left of the separator will be the key and the 

369 value to the right of the separator will be the value. 

370 If `tuple` then the returned object will be a tuple. Each item in the 

371 tuple will be 2-item tuple, the first item will be the value to the 

372 left of the separator and the second item will be the value to the 

373 right. By default `dict`. 

374 default_key : `Any` 

375 The key to use if a value is passed that is not a key-value pair. 

376 (Passing values that are not key-value pairs requires 

377 ``unseparated_okay`` to be `True`.) 

378 reverse_kv : bool 

379 If true then for each item in values, the value to the left of the 

380 separator is treated as the value and the value to the right of the 

381 separator is treated as the key. By default False. 

382 add_to_default : `bool`, optional 

383 If True, then passed-in values will not overwrite the default value 

384 unless the ``return_type`` is `dict` and passed-in value(s) have the 

385 same key(s) as the default value. 

386 

387 Returns 

388 ------- 

389 values : `dict` [`str`, `str`] 

390 The passed-in values in dict form. 

391 

392 Raises 

393 ------ 

394 `click.ClickException` 

395 Raised if the separator is not found in an entry, or if duplicate keys 

396 are encountered. 

397 """ 

398 

399 def norm(val): 

400 """If `normalize` is True and `choice` is not `None`, find the value 

401 in the available choices and return the value as spelled in the 

402 choices. 

403 

404 Assumes that val exists in choices; `split_kv` uses the `choice` 

405 instance to verify val is a valid choice. 

406 """ 

407 if normalize and choice is not None: 

408 v = val.casefold() 

409 for opt in choice.choices: 

410 if opt.casefold() == v: 

411 return opt 

412 return val 

413 

414 class RetDict: 

415 def __init__(self): 

416 self.ret = {} 

417 

418 def add(self, key, val): 

419 if reverse_kv: 

420 key, val = val, key 

421 self.ret[key] = val 

422 

423 def get(self): 

424 return self.ret 

425 

426 class RetTuple: 

427 def __init__(self): 

428 self.ret = [] 

429 

430 def add(self, key, val): 

431 if reverse_kv: 

432 key, val = val, key 

433 self.ret.append((key, val)) 

434 

435 def get(self): 

436 return tuple(self.ret) 

437 

438 if separator in (",", " "): 

439 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.") 

440 vals = values # preserve the original argument for error reporting below. 

441 

442 if add_to_default: 

443 default = param.get_default(context) 

444 if default: 

445 vals = itertools.chain(default, vals) 

446 

447 if return_type is dict: 

448 ret = RetDict() 

449 elif return_type is tuple: 

450 ret = RetTuple() 

451 else: 

452 raise click.ClickException( 

453 message=f"Internal error: invalid return type '{return_type}' for split_kv." 

454 ) 

455 if multiple: 

456 vals = split_commas(context, param, vals) 

457 for val in ensure_iterable(vals): 

458 if unseparated_okay and separator not in val: 

459 if choice is not None: 

460 choice(val) # will raise if val is an invalid choice 

461 ret.add(default_key, norm(val)) 

462 else: 

463 try: 

464 k, v = val.split(separator) 

465 if choice is not None: 

466 choice(v) # will raise if val is an invalid choice 

467 except ValueError as e: 

468 raise click.ClickException( 

469 message=f"Could not parse key-value pair '{val}' using separator '{separator}', " 

470 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}" 

471 ) 

472 ret.add(k, norm(v)) 

473 return ret.get() 

474 

475 

476def to_upper(context, param, value): 

477 """Convert a value to upper case. 

478 

479 Parameters 

480 ---------- 

481 context : click.Context 

482 

483 values : string 

484 The value to be converted. 

485 

486 Returns 

487 ------- 

488 string 

489 A copy of the passed-in value, converted to upper case. 

490 """ 

491 return value.upper() 

492 

493 

494def unwrap(val): 

495 """Remove newlines and leading whitespace from a multi-line string with 

496 a consistent indentation level. 

497 

498 The first line of the string may be only a newline or may contain text 

499 followed by a newline, either is ok. After the first line, each line must 

500 begin with a consistant amount of whitespace. So, content of a 

501 triple-quoted string may begin immediately after the quotes, or the string 

502 may start with a newline. Each line after that must be the same amount of 

503 indentation/whitespace followed by text and a newline. The last line may 

504 end with a new line but is not required to do so. 

505 

506 Parameters 

507 ---------- 

508 val : `str` 

509 The string to change. 

510 

511 Returns 

512 ------- 

513 strippedString : `str` 

514 The string with newlines, indentation, and leading and trailing 

515 whitespace removed. 

516 """ 

517 

518 def splitSection(val): 

519 if not val.startswith("\n"): 519 ↛ 523line 519 didn't jump to line 523, because the condition on line 519 was never false

520 firstLine, _, val = val.partition("\n") 

521 firstLine += " " 

522 else: 

523 firstLine = "" 

524 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip() 

525 

526 return "\n\n".join([splitSection(s) for s in val.split("\n\n")]) 

527 

528 

529class option_section: # noqa: N801 

530 """Decorator to add a section label between options in the help text of a 

531 command. 

532 

533 Parameters 

534 ---------- 

535 sectionText : `str` 

536 The text to print in the section identifier. 

537 """ 

538 

539 def __init__(self, sectionText): 

540 self.sectionText = "\n" + sectionText 

541 

542 def __call__(self, f): 

543 # Generate a parameter declaration that will be unique for this 

544 # section. 

545 return click.option( 

546 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection 

547 )(f) 

548 

549 

550class MWPath(click.Path): 

551 """Overrides click.Path to implement file-does-not-exist checking. 

552 

553 Changes the definition of ``exists` so that `True` indicates the location 

554 (file or directory) must exist, `False` indicates the location must *not* 

555 exist, and `None` indicates that the file may exist or not. The standard 

556 definition for the `click.Path` ``exists`` parameter is that for `True` a 

557 location must exist, but `False` means it is not required to exist (not 

558 that it is required to not exist). 

559 

560 Parameters 

561 ---------- 

562 exists : `True`, `False`, or `None` 

563 If `True`, the location (file or directory) indicated by the caller 

564 must exist. If `False` the location must not exist. If `None`, the 

565 location may exist or not. 

566 

567 For other parameters see `click.Path`. 

568 """ 

569 

570 def __init__( 

571 self, 

572 exists=None, 

573 file_okay=True, 

574 dir_okay=True, 

575 writable=False, 

576 readable=True, 

577 resolve_path=False, 

578 allow_dash=False, 

579 path_type=None, 

580 ): 

581 self.mustNotExist = exists is False 

582 if exists is None: 582 ↛ 584line 582 didn't jump to line 584, because the condition on line 582 was never false

583 exists = False 

584 super().__init__( 

585 exists=exists, 

586 file_okay=file_okay, 

587 dir_okay=dir_okay, 

588 writable=writable, 

589 readable=readable, 

590 resolve_path=resolve_path, 

591 allow_dash=allow_dash, 

592 path_type=path_type, 

593 ) 

594 

595 def convert(self, value, param, ctx): 

596 """Called by click.ParamType to "convert values through types". 

597 `click.Path` uses this step to verify Path conditions.""" 

598 if self.mustNotExist and os.path.exists(value): 

599 self.fail(f'Path "{value}" should not exist.') 

600 return super().convert(value, param, ctx) 

601 

602 

603class MWOption(click.Option): 

604 """Overrides click.Option with desired behaviors.""" 

605 

606 def make_metavar(self): 

607 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

608 help menu. Adds a space and an elipsis after the metavar name if 

609 the option accepts multiple inputs, otherwise defers to the base 

610 implementation. 

611 

612 By default click does not add an elipsis when multiple is True and 

613 nargs is 1. And when nargs does not equal 1 click adds an elipsis 

614 without a space between the metavar and the elipsis, but we prefer a 

615 space between. 

616 

617 Does not get called for some option types (e.g. flag) so metavar 

618 transformation that must apply to all types should be applied in 

619 get_help_record. 

620 """ 

621 metavar = super().make_metavar() 

622 if self.multiple and self.nargs == 1: 

623 metavar += " ..." 

624 elif self.nargs != 1: 

625 metavar = f"{metavar[:-3]} ..." 

626 return metavar 

627 

628 

629class MWArgument(click.Argument): 

630 """Overrides click.Argument with desired behaviors.""" 

631 

632 def make_metavar(self): 

633 """Overrides `click.Option.make_metavar`. Makes the metavar for the 

634 help menu. Always adds a space and an elipsis (' ...') after the 

635 metavar name if the option accepts multiple inputs. 

636 

637 By default click adds an elipsis without a space between the metavar 

638 and the elipsis, but we prefer a space between. 

639 

640 Returns 

641 ------- 

642 metavar : `str` 

643 The metavar value. 

644 """ 

645 metavar = super().make_metavar() 

646 if self.nargs != 1: 

647 metavar = f"{metavar[:-3]} ..." 

648 return metavar 

649 

650 

651class OptionSection(MWOption): 

652 """Implements an Option that prints a section label in the help text and 

653 does not pass any value to the command function. 

654 

655 This class does a bit of hackery to add a section label to a click command 

656 help output: first, `expose_value` is set to `False` so that no value is 

657 passed to the command function. Second, this class overrides 

658 `click.Option.get_help_record` to return the section label string without 

659 any prefix so that it stands out as a section label. 

660 

661 This class overrides the hidden attribute because our documentation build 

662 tool, sphinx-click, implements its own `get_help_record` function which 

663 builds the record from other option values (e.g. `name`, `opts`), which 

664 breaks the hack we use to make `get_help_record` only return the 

665 `sectionText`. Fortunately, Click gets the value of `hidden` inside the 

666 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before 

667 entering its `_get_help_record` function. So, making the hidden property 

668 return True hides this option from sphinx-click, while allowing the section 

669 text to be returned by our `get_help_record` method when using Click. 

670 

671 The intention for this implementation is to do minimally invasive overrides 

672 of the click classes so as to be robust and easy to fix if the click 

673 internals change. 

674 

675 Parameters 

676 ---------- 

677 sectionName : `str` 

678 The parameter declaration for this option. It is not shown to the user, 

679 it must be unique within the command. If using the `section` decorator 

680 to add a section to a command's options, the section name is 

681 auto-generated. 

682 sectionText : `str` 

683 The text to print in the section identifier. 

684 """ 

685 

686 @property 

687 def hidden(self): 

688 return True 

689 

690 @hidden.setter 

691 def hidden(self, val): 

692 pass 

693 

694 def __init__(self, sectionName, sectionText): 

695 super().__init__(sectionName, expose_value=False) 

696 self.sectionText = sectionText 

697 

698 def get_help_record(self, ctx): 

699 return (self.sectionText, "") 

700 

701 

702class MWOptionDecorator: 

703 """Wraps the click.option decorator to enable shared options to be declared 

704 and allows inspection of the shared option. 

705 """ 

706 

707 def __init__(self, *param_decls, **kwargs): 

708 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) 

709 opt = click.Option(param_decls, **kwargs) 

710 self._name = opt.name 

711 self._opts = opt.opts 

712 

713 def name(self): 

714 """Get the name that will be passed to the command function for this 

715 option.""" 

716 return self._name 

717 

718 def opts(self): 

719 """Get the flags that will be used for this option on the command 

720 line.""" 

721 return self._opts 

722 

723 @property 

724 def help(self): 

725 """Get the help text for this option. Returns an empty string if no 

726 help was defined.""" 

727 return self.partialOpt.keywords.get("help", "") 

728 

729 def __call__(self, *args, **kwargs): 

730 return self.partialOpt(*args, **kwargs) 

731 

732 

733class MWArgumentDecorator: 

734 """Wraps the click.argument decorator to enable shared arguments to be 

735 declared.""" 

736 

737 def __init__(self, *param_decls, **kwargs): 

738 self._helpText = kwargs.pop("help", None) 

739 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs) 

740 

741 def __call__(self, *args, help=None, **kwargs): 

742 def decorator(f): 

743 if help is not None: 

744 self._helpText = help 

745 if self._helpText: 

746 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText) 

747 return self.partialArg(*args, **kwargs)(f) 

748 

749 return decorator 

750 

751 

752class MWCommand(click.Command): 

753 """Command subclass that stores a copy of the args list for use by the 

754 command.""" 

755 

756 extra_epilog = None 

757 

758 def __init__(self, *args, **kwargs): 

759 # wrap callback method with catch_and_exit decorator 

760 callback = kwargs.get("callback") 

761 if callback is not None: 761 ↛ 764line 761 didn't jump to line 764, because the condition on line 761 was never false

762 kwargs = kwargs.copy() 

763 kwargs["callback"] = catch_and_exit(callback) 

764 super().__init__(*args, **kwargs) 

765 

766 def _capture_args(self, ctx, args): 

767 """Capture the command line options and arguments. 

768 

769 See details about what is captured and the order in which it is stored 

770 in the documentation of `MWCtxObj`. 

771 

772 Parameters 

773 ---------- 

774 ctx : `click.Context` 

775 The current Context. 

776 args : `list` [`str`] 

777 The list of arguments from the command line, split at spaces but 

778 not at separators (like "="). 

779 """ 

780 parser = self.make_parser(ctx) 

781 opts, _, param_order = parser.parse_args(args=list(args)) 

782 # `param_order` is a list of click.Option and click.Argument, there is 

783 # one item for each time the Option or Argument was used on the 

784 # command line. Options will precede Arguments, within each sublist 

785 # they are in the order they were used on the command line. Note that 

786 # click.Option and click.Argument do not contain the value from the 

787 # command line; values are in `opts`. 

788 # 

789 # `opts` is a dict where the key is the argument name to the 

790 # click.Command function, this name matches the `click.Option.name` or 

791 # `click.Argument.name`. For Options, an item will only be present if 

792 # the Option was used on the command line. For Arguments, an item will 

793 # always be present and if no value was provided on the command line 

794 # the value will be `None`. If the option accepts multiple values, the 

795 # value in `opts` is a tuple, otherwise it is a single item. 

796 next_idx = Counter() 

797 captured_args = [] 

798 for param in param_order: 

799 if isinstance(param, click.Option): 

800 if param.multiple: 

801 val = opts[param.name][next_idx[param.name]] 

802 next_idx[param.name] += 1 

803 else: 

804 val = opts[param.name] 

805 if param.is_flag: 

806 # Bool options store their True flags in opts and their 

807 # False flags in secondary_opts. 

808 if val: 

809 flag = max(param.opts, key=len) 

810 else: 

811 flag = max(param.secondary_opts, key=len) 

812 captured_args.append(flag) 

813 else: 

814 captured_args.append(max(param.opts, key=len)) 

815 captured_args.append(val) 

816 elif isinstance(param, click.Argument): 

817 if (opt := opts[param.name]) is not None: 

818 captured_args.append(opt) 

819 else: 

820 assert False # All parameters should be an Option or an Argument. 

821 MWCtxObj.getFrom(ctx).args = captured_args 

822 

823 def parse_args(self, ctx, args): 

824 """Given a context and a list of arguments this creates the parser and 

825 parses the arguments, then modifies the context as necessary. This is 

826 automatically invoked by make_context(). 

827 

828 This function overrides `click.Command.parse_args`. 

829 

830 The call to `_capture_args` in this override stores the arguments 

831 (option names, option value, and argument values) that were used by the 

832 caller on the command line in the context object. These stored 

833 arugments can be used by the command function, e.g. to process options 

834 in the order they appeared on the command line (pipetask uses this 

835 feature to create pipeline actions in an order from different options). 

836 

837 Parameters 

838 ---------- 

839 ctx : `click.core.Context` 

840 The current Context.ß 

841 args : `list` [`str`] 

842 The list of arguments from the command line, split at spaces but 

843 not at separators (like "="). 

844 """ 

845 self._capture_args(ctx, args) 

846 super().parse_args(ctx, args) 

847 

848 @property 

849 def epilog(self): 

850 """Override the epilog attribute to add extra_epilog (if defined by a 

851 subclass) to the end of any epilog provided by a subcommand. 

852 """ 

853 ret = self._epilog if self._epilog else "" 

854 if self.extra_epilog: 

855 if ret: 

856 ret += "\n\n" 

857 ret += self.extra_epilog 

858 return ret 

859 

860 @epilog.setter 

861 def epilog(self, val): 

862 self._epilog = val 

863 

864 

865class ButlerCommand(MWCommand): 

866 """Command subclass with butler-command specific overrides.""" 

867 

868 extra_epilog = "See 'butler --help' for more options." 

869 

870 

871class OptionGroup: 

872 """Base class for an option group decorator. Requires the option group 

873 subclass to have a property called `decorator`.""" 

874 

875 def __call__(self, f): 

876 for decorator in reversed(self.decorators): 

877 f = decorator(f) 

878 return f 

879 

880 

881class MWCtxObj: 

882 """Helper object for managing the `click.Context.obj` parameter, allows 

883 obj data to be managed in a consistent way. 

884 

885 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to 

886 initialize the obj if needed and return a new or existing `MWCtxObj`. 

887 

888 The `args` attribute contains a list of options, option values, and 

889 argument values that is similar to the list of arguments and options that 

890 were passed in on the command line, with differences noted below: 

891 

892 * Option namess and option values are first in the list, and argument 

893 values come last. The order of options and option values is preserved 

894 within the options. The order of argument values is preserved. 

895 

896 * The longest option name is used for the option in the `args` list, e.g. 

897 if an option accepts both short and long names "-o / --option" and the 

898 short option name "-o" was used on the command line, the longer name will 

899 be the one that appears in `args`. 

900 

901 * A long option name (which begins with two dashes "--") and its value may 

902 be separated by an equal sign; the name and value are split at the equal 

903 sign and it is removed. In `args`, the option is in one list item, and 

904 the option value (without the equal sign) is in the next list item. e.g. 

905 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in 

906 `args`. 

907 

908 * A short option name, (which begins with one dash "-") and its value are 

909 split immediately after the short option name, and if there is 

910 whitespace between the short option name and its value it is removed. 

911 Everything after the short option name (excluding whitespace) is included 

912 in the value. If the `Option` has a long name, the long name will be used 

913 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo" 

914 become `["--option", "foo"]`, and (note!) "-o=foo" will become 

915 `["--option", "=foo"]` (because everything after the short option name, 

916 except whitespace, is used for the value (as is standard with unix 

917 command line tools). 

918 

919 Attributes 

920 ---------- 

921 args : `list` [`str`] 

922 A list of options, option values, and arguments simialr to those that 

923 were passed in on the command line. See comments about captured options 

924 & arguments above. 

925 """ 

926 

927 def __init__(self): 

928 

929 self.args = None 

930 

931 @staticmethod 

932 def getFrom(ctx): 

933 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return 

934 the new or already existing `MWCtxObj`.""" 

935 if ctx.obj is not None: 

936 return ctx.obj 

937 ctx.obj = MWCtxObj() 

938 return ctx.obj 

939 

940 

941def yaml_presets(ctx, param, value): 

942 """Click callback that reads additional values from the supplied 

943 YAML file. 

944 

945 Parameters 

946 ---------- 

947 ctx : `click.context` 

948 The context for the click operation. Used to extract the subcommand 

949 name and translate option & argument names. 

950 param : `str` 

951 The parameter name. 

952 value : `object` 

953 The value of the parameter. 

954 """ 

955 

956 def _name_for_option(ctx: click.Context, option: str) -> str: 

957 """Use a CLI option name to find the name of the argument to the 

958 command function. 

959 

960 Parameters 

961 ---------- 

962 ctx : `click.Context` 

963 The context for the click operation. 

964 option : `str` 

965 The option/argument name from the yaml file. 

966 

967 Returns 

968 ------- 

969 name : str 

970 The name of the argument to use when calling the click.command 

971 function, as it should appear in the `ctx.default_map`. 

972 

973 Raises 

974 ------ 

975 RuntimeError 

976 Raised if the option name from the yaml file does not exist in the 

977 command parameters. This catches misspellings and incorrect useage 

978 in the yaml file. 

979 """ 

980 for param in ctx.command.params: 

981 # Remove leading dashes: they are not used for option names in the 

982 # yaml file. 

983 if option in [opt.lstrip("-") for opt in param.opts]: 

984 return param.name 

985 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}") 

986 

987 ctx.default_map = ctx.default_map or {} 

988 cmd_name = ctx.info_name 

989 if value: 

990 try: 

991 overrides = _read_yaml_presets(value, cmd_name) 

992 options = list(overrides.keys()) 

993 for option in options: 

994 name = _name_for_option(ctx, option) 

995 if name == option: 

996 continue 

997 overrides[name] = overrides.pop(option) 

998 except Exception as e: 

999 raise click.BadOptionUsage( 

1000 option_name=param.name, 

1001 message=f"Error reading overrides file: {e}", 

1002 ctx=ctx, 

1003 ) 

1004 # Override the defaults for this subcommand 

1005 ctx.default_map.update(overrides) 

1006 return 

1007 

1008 

1009def _read_yaml_presets(file_uri, cmd_name): 

1010 """Read file command line overrides from YAML config file. 

1011 

1012 Parameters 

1013 ---------- 

1014 file_uri : `str` 

1015 URI of override YAML file containing the command line overrides. 

1016 They should be grouped by command name. 

1017 cmd_name : `str` 

1018 The subcommand name that is being modified. 

1019 

1020 Returns 

1021 ------- 

1022 overrides : `dict` of [`str`, Any] 

1023 The relevant command line options read from the override file. 

1024 """ 

1025 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri) 

1026 config = Config(file_uri) 

1027 return config[cmd_name] 

1028 

1029 

1030def sortAstropyTable(table, dimensions, sort_first=None): 

1031 """Sort an astropy table, with prioritization given to columns in this 

1032 order: 

1033 1. the provided named columns 

1034 2. spatial and temporal columns 

1035 3. the rest of the columns 

1036 

1037 The table is sorted in-place, and is also returned for convenience. 

1038 

1039 Parameters 

1040 ---------- 

1041 table : `astropy.table.Table` 

1042 The table to sort 

1043 dimensions : `list` [``Dimension``] 

1044 The dimensions of the dataIds in the table (the dimensions should be 

1045 the same for all the dataIds). Used to determine if the column is 

1046 spatial, temporal, or neither. 

1047 sort_first : `list` [`str`] 

1048 The names of columns that should be sorted first, before spatial and 

1049 temporal columns. 

1050 

1051 Returns 

1052 ------- 

1053 `astropy.table.Table` 

1054 For convenience, the table that has been sorted. 

1055 """ 

1056 # For sorting we want to ignore the id 

1057 # We also want to move temporal or spatial dimensions earlier 

1058 sort_first = sort_first or [] 

1059 sort_early = [] 

1060 sort_late = [] 

1061 for dim in dimensions: 

1062 if dim.spatial or dim.temporal: 

1063 sort_early.extend(dim.required.names) 

1064 else: 

1065 sort_late.append(str(dim)) 

1066 sort_keys = sort_first + sort_early + sort_late 

1067 # The required names above means that we have the possibility of 

1068 # repeats of sort keys. Now have to remove them 

1069 # (order is retained by dict creation). 

1070 sort_keys = list(dict.fromkeys(sort_keys).keys()) 

1071 

1072 table.sort(sort_keys) 

1073 return table 

1074 

1075 

1076def catch_and_exit(func): 

1077 """Decorator which catches all exceptions, prints an exception traceback 

1078 and signals click to exit. 

1079 """ 

1080 

1081 @wraps(func) 

1082 def inner(*args, **kwargs): 

1083 try: 

1084 func(*args, **kwargs) 

1085 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort): 

1086 # this is handled by click itself 

1087 raise 

1088 except Exception: 

1089 exc_type, exc_value, exc_tb = sys.exc_info() 

1090 if exc_tb.tb_next: 

1091 # do not show this decorator in traceback 

1092 exc_tb = exc_tb.tb_next 

1093 log.exception( 

1094 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb) 

1095 ) 

1096 # tell click to stop, this never returns. 

1097 click.get_current_context().exit(1) 

1098 

1099 return inner