Coverage for python/lsst/daf/butler/cli/utils.py: 32%
353 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-10-26 15:15 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-10-26 15:15 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "astropyTablesToStr",
25 "printAstropyTables",
26 "textTypeStr",
27 "LogCliRunner",
28 "clickResultMsg",
29 "command_test_env",
30 "addArgumentHelp",
31 "split_commas",
32 "split_kv",
33 "to_upper",
34 "unwrap",
35 "option_section",
36 "MWPath",
37 "MWOption",
38 "MWArgument",
39 "OptionSection",
40 "MWOptionDecorator",
41 "MWArgumentDecorator",
42 "MWCommand",
43 "ButlerCommand",
44 "OptionGroup",
45 "MWCtxObj",
46 "yaml_presets",
47 "sortAstropyTable",
48 "catch_and_exit",
49)
52import itertools
53import logging
54import os
55import re
56import sys
57import textwrap
58import traceback
59import uuid
60import warnings
61from collections import Counter
62from collections.abc import Callable, Iterable, Iterator
63from contextlib import contextmanager
64from functools import partial, wraps
65from typing import TYPE_CHECKING, Any
66from unittest.mock import patch
68import click
69import click.exceptions
70import click.testing
71import yaml
72from lsst.utils.iteration import ensure_iterable
74from ..core.config import Config
75from .cliLog import CliLog
77if TYPE_CHECKING: 77 ↛ 78line 77 didn't jump to line 78, because the condition on line 77 was never true
78 from astropy.table import Table
79 from lsst.daf.butler import Dimension
81log = logging.getLogger(__name__)
83# This is used as the metavar argument to Options that accept multiple string
84# inputs, which may be comma-separarated. For example:
85# --my-opt foo,bar --my-opt baz.
86# Other arguments to the Option should include multiple=true and
87# callback=split_kv.
88typeStrAcceptsMultiple = "TEXT ..."
89typeStrAcceptsSingle = "TEXT"
91# The standard help string for the --where option when it takes a WHERE clause.
92where_help = (
93 "A string expression similar to a SQL WHERE clause. May involve any column of a "
94 "dimension table or a dimension name as a shortcut for the primary key column of a "
95 "dimension table."
96)
99def astropyTablesToStr(tables: list[Table]) -> str:
100 """Render astropy tables to string as they are displayed in the CLI.
102 Output formatting matches ``printAstropyTables``.
103 """
104 ret = ""
105 for table in tables:
106 ret += "\n"
107 table.pformat_all()
108 ret += "\n"
109 return ret
112def printAstropyTables(tables: list[Table]) -> None:
113 """Print astropy tables to be displayed in the CLI.
115 Output formatting matches ``astropyTablesToStr``.
116 """
117 for table in tables:
118 print("")
119 table.pprint_all()
120 print("")
123def textTypeStr(multiple: bool) -> str:
124 """Get the text type string for CLI help documentation.
126 Parameters
127 ----------
128 multiple : `bool`
129 True if multiple text values are allowed, False if only one value is
130 allowed.
132 Returns
133 -------
134 textTypeStr : `str`
135 The type string to use.
136 """
137 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
140class LogCliRunner(click.testing.CliRunner):
141 """A test runner to use when the logging system will be initialized by code
142 under test, calls CliLog.resetLog(), which undoes any logging setup that
143 was done with the CliLog interface.
145 lsst.log modules can not be set back to an uninitialized state (python
146 logging modules can be set back to NOTSET), instead they are set to
147 `CliLog.defaultLsstLogLevel`."""
149 def invoke(self, *args: Any, **kwargs: Any) -> Any:
150 result = super().invoke(*args, **kwargs)
151 CliLog.resetLog()
152 return result
155def clickResultMsg(result: click.testing.Result) -> str:
156 """Get a standard assert message from a click result
158 Parameters
159 ----------
160 result : click.testing.Result
161 The result object returned from click.testing.CliRunner.invoke
163 Returns
164 -------
165 msg : `str`
166 The message string.
167 """
168 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
169 if result.exception:
170 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
171 return msg
174@contextmanager
175def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
176 """A context manager that creates (and then cleans up) an environment that
177 provides a CLI plugin command with the given name.
179 Parameters
180 ----------
181 runner : click.testing.CliRunner
182 The test runner to use to create the isolated filesystem.
183 commandModule : `str`
184 The importable module that the command can be imported from.
185 commandName : `str`
186 The name of the command being published to import.
187 """
188 with runner.isolated_filesystem():
189 with open("resources.yaml", "w") as f:
190 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
191 # Add a colon to the end of the path on the next line, this tests the
192 # case where the lookup in LoaderCLI._getPluginList generates an empty
193 # string in one of the list entries and verifies that the empty string
194 # is properly stripped out.
195 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
196 yield
199def addArgumentHelp(doc: str | None, helpText: str) -> str:
200 """Add a Click argument's help message to a function's documentation.
202 This is needed because click presents arguments in the order the argument
203 decorators are applied to a function, top down. But, the evaluation of the
204 decorators happens bottom up, so if arguments just append their help to the
205 function's docstring, the argument descriptions appear in reverse order
206 from the order they are applied in.
208 Parameters
209 ----------
210 doc : `str`
211 The function's docstring.
212 helpText : `str`
213 The argument's help string to be inserted into the function's
214 docstring.
216 Returns
217 -------
218 doc : `str`
219 Updated function documentation.
220 """
221 if doc is None: 221 ↛ 222line 221 didn't jump to line 222, because the condition on line 221 was never true
222 doc = helpText
223 else:
224 # See click documentation for details:
225 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
226 # In short, text for the click command help can be truncated by putting
227 # "\f" in the docstring, everything after it should be removed
228 if "\f" in doc: 228 ↛ 229line 228 didn't jump to line 229, because the condition on line 228 was never true
229 doc = doc.split("\f")[0]
231 doclines = doc.splitlines()
232 # The function's docstring may span multiple lines, so combine the
233 # docstring from all the first lines until a blank line is encountered.
234 # (Lines after the first blank line will be argument help.)
235 while len(doclines) > 1 and doclines[1]:
236 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
237 # Add standard indent to help text for proper alignment with command
238 # function documentation:
239 helpText = " " + helpText
240 doclines.insert(1, helpText)
241 doclines.insert(1, "\n")
242 doc = "\n".join(doclines)
243 return doc
246def split_commas(
247 context: click.Context, param: click.core.Option, values: str | Iterable[str] | None
248) -> tuple[str, ...]:
249 """Process a tuple of values, where each value may contain comma-separated
250 values, and return a single list of all the passed-in values.
252 This function can be passed to the 'callback' argument of a click.option to
253 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
254 the comma is inside ``[]`` there will be no splitting.
256 Parameters
257 ----------
258 context : `click.Context` or `None`
259 The current execution context. Unused, but Click always passes it to
260 callbacks.
261 param : `click.core.Option` or `None`
262 The parameter being handled. Unused, but Click always passes it to
263 callbacks.
264 values : iterable of `str` or `str`
265 All the values passed for this option. Strings may contain commas,
266 which will be treated as delimiters for separate values unless they
267 are within ``[]``.
269 Returns
270 -------
271 results : `tuple` [`str`]
272 The passed in values separated by commas where appropriate and
273 combined into a single tuple.
274 """
275 if values is None:
276 return tuple()
277 valueList = []
278 for value in ensure_iterable(values):
279 # If we have [, or ,] we do the slow split. If square brackets
280 # are not matching then that is likely a typo that should result
281 # in a warning.
282 opens = "["
283 closes = "]"
284 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
285 in_parens = False
286 current = ""
287 for c in value:
288 if c == opens:
289 if in_parens:
290 warnings.warn(
291 f"Found second opening {opens} without corresponding closing {closes}"
292 f" in {value!r}",
293 stacklevel=2,
294 )
295 in_parens = True
296 elif c == closes:
297 if not in_parens:
298 warnings.warn(
299 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
300 stacklevel=2,
301 )
302 in_parens = False
303 elif c == ",":
304 if not in_parens:
305 # Split on this comma.
306 valueList.append(current)
307 current = ""
308 continue
309 current += c
310 if in_parens:
311 warnings.warn(
312 f"Found opening {opens} that was never closed in {value!r}",
313 stacklevel=2,
314 )
315 if current:
316 valueList.append(current)
317 else:
318 # Use efficient split since no parens.
319 valueList.extend(value.split(","))
320 return tuple(valueList)
323def split_kv(
324 context: click.Context,
325 param: click.core.Option,
326 values: list[str],
327 *,
328 choice: click.Choice | None = None,
329 multiple: bool = True,
330 normalize: bool = False,
331 separator: str = "=",
332 unseparated_okay: bool = False,
333 return_type: type[dict] | type[tuple] = dict,
334 default_key: str = "",
335 reverse_kv: bool = False,
336 add_to_default: bool = False,
337) -> dict[str, str] | tuple[tuple[str, str], ...]:
338 """Process a tuple of values that are key-value pairs separated by a given
339 separator. Multiple pairs may be comma separated. Return a dictionary of
340 all the passed-in values.
342 This function can be passed to the 'callback' argument of a click.option to
343 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
345 Parameters
346 ----------
347 context : `click.Context` or `None`
348 The current execution context. Unused, but Click always passes it to
349 callbacks.
350 param : `click.core.Option` or `None`
351 The parameter being handled. Unused, but Click always passes it to
352 callbacks.
353 values : [`str`]
354 All the values passed for this option. Strings may contain commas,
355 which will be treated as delimiters for separate values.
356 choice : `click.Choice`, optional
357 If provided, verify each value is a valid choice using the provided
358 `click.Choice` instance. If None, no verification will be done. By
359 default None
360 multiple : `bool`, optional
361 If true, the value may contain multiple comma-separated values. By
362 default True.
363 normalize : `bool`, optional
364 If True and `choice.case_sensitive == False`, normalize the string the
365 user provided to match the choice's case. By default False.
366 separator : str, optional
367 The character that separates key-value pairs. May not be a comma or an
368 empty space (for space separators use Click's default implementation
369 for tuples; `type=(str, str)`). By default "=".
370 unseparated_okay : `bool`, optional
371 If True, allow values that do not have a separator. They will be
372 returned in the values dict as a tuple of values in the key '', that
373 is: `values[''] = (unseparated_values, )`. By default False.
374 return_type : `type`, must be `dict` or `tuple`
375 The type of the value that should be returned.
376 If `dict` then the returned object will be a dict, for each item in
377 values, the value to the left of the separator will be the key and the
378 value to the right of the separator will be the value.
379 If `tuple` then the returned object will be a tuple. Each item in the
380 tuple will be 2-item tuple, the first item will be the value to the
381 left of the separator and the second item will be the value to the
382 right. By default `dict`.
383 default_key : `Any`
384 The key to use if a value is passed that is not a key-value pair.
385 (Passing values that are not key-value pairs requires
386 ``unseparated_okay`` to be `True`.)
387 reverse_kv : bool
388 If true then for each item in values, the value to the left of the
389 separator is treated as the value and the value to the right of the
390 separator is treated as the key. By default False.
391 add_to_default : `bool`, optional
392 If True, then passed-in values will not overwrite the default value
393 unless the ``return_type`` is `dict` and passed-in value(s) have the
394 same key(s) as the default value.
396 Returns
397 -------
398 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...]
399 The passed-in values in dict form or tuple form.
401 Raises
402 ------
403 `click.ClickException`
404 Raised if the separator is not found in an entry, or if duplicate keys
405 are encountered.
406 """
408 def norm(val: str) -> str:
409 """If `normalize` is True and `choice` is not `None`, find the value
410 in the available choices and return the value as spelled in the
411 choices.
413 Assumes that val exists in choices; `split_kv` uses the `choice`
414 instance to verify val is a valid choice.
415 """
416 if normalize and choice is not None:
417 v = val.casefold()
418 for opt in choice.choices:
419 if opt.casefold() == v:
420 return opt
421 return val
423 class RetDict:
424 def __init__(self) -> None:
425 self.ret: dict[str, str] = {}
427 def add(self, key: str, val: str) -> None:
428 if reverse_kv:
429 key, val = val, key
430 self.ret[key] = val
432 def get(self) -> dict[str, str]:
433 return self.ret
435 class RetTuple:
436 def __init__(self) -> None:
437 self.ret: list[tuple[str, str]] = []
439 def add(self, key: str, val: str) -> None:
440 if reverse_kv:
441 key, val = val, key
442 self.ret.append((key, val))
444 def get(self) -> tuple[tuple[str, str], ...]:
445 return tuple(self.ret)
447 if separator in (",", " "):
448 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
449 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
451 if add_to_default:
452 default = param.get_default(context)
453 if default:
454 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy
456 ret: RetDict | RetTuple
457 if return_type is dict:
458 ret = RetDict()
459 elif return_type is tuple:
460 ret = RetTuple()
461 else:
462 raise click.ClickException(
463 message=f"Internal error: invalid return type '{return_type}' for split_kv."
464 )
465 if multiple:
466 vals = split_commas(context, param, vals)
467 for val in ensure_iterable(vals):
468 if unseparated_okay and separator not in val:
469 if choice is not None:
470 choice(val) # will raise if val is an invalid choice
471 ret.add(default_key, norm(val))
472 else:
473 try:
474 k, v = val.split(separator)
475 if choice is not None:
476 choice(v) # will raise if val is an invalid choice
477 except ValueError as e:
478 raise click.ClickException(
479 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
480 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
481 )
482 ret.add(k, norm(v))
483 return ret.get()
486def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
487 """Convert a value to upper case.
489 Parameters
490 ----------
491 context : click.Context
493 values : string
494 The value to be converted.
496 Returns
497 -------
498 string
499 A copy of the passed-in value, converted to upper case.
500 """
501 return value.upper()
504def unwrap(val: str) -> str:
505 """Remove newlines and leading whitespace from a multi-line string with
506 a consistent indentation level.
508 The first line of the string may be only a newline or may contain text
509 followed by a newline, either is ok. After the first line, each line must
510 begin with a consistant amount of whitespace. So, content of a
511 triple-quoted string may begin immediately after the quotes, or the string
512 may start with a newline. Each line after that must be the same amount of
513 indentation/whitespace followed by text and a newline. The last line may
514 end with a new line but is not required to do so.
516 Parameters
517 ----------
518 val : `str`
519 The string to change.
521 Returns
522 -------
523 strippedString : `str`
524 The string with newlines, indentation, and leading and trailing
525 whitespace removed.
526 """
528 def splitSection(val: str) -> str:
529 if not val.startswith("\n"): 529 ↛ 533line 529 didn't jump to line 533, because the condition on line 529 was never false
530 firstLine, _, val = val.partition("\n")
531 firstLine += " "
532 else:
533 firstLine = ""
534 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
536 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
539class option_section: # noqa: N801
540 """Decorator to add a section label between options in the help text of a
541 command.
543 Parameters
544 ----------
545 sectionText : `str`
546 The text to print in the section identifier.
547 """
549 def __init__(self, sectionText: str) -> None:
550 self.sectionText = "\n" + sectionText
552 def __call__(self, f: Any) -> click.Option:
553 # Generate a parameter declaration that will be unique for this
554 # section.
555 return click.option(
556 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
557 )(f)
560class MWPath(click.Path):
561 """Overrides click.Path to implement file-does-not-exist checking.
563 Changes the definition of ``exists` so that `True` indicates the location
564 (file or directory) must exist, `False` indicates the location must *not*
565 exist, and `None` indicates that the file may exist or not. The standard
566 definition for the `click.Path` ``exists`` parameter is that for `True` a
567 location must exist, but `False` means it is not required to exist (not
568 that it is required to not exist).
570 Parameters
571 ----------
572 exists : `True`, `False`, or `None`
573 If `True`, the location (file or directory) indicated by the caller
574 must exist. If `False` the location must not exist. If `None`, the
575 location may exist or not.
577 For other parameters see `click.Path`.
578 """
580 def __init__(
581 self,
582 exists: bool | None = None,
583 file_okay: bool = True,
584 dir_okay: bool = True,
585 writable: bool = False,
586 readable: bool = True,
587 resolve_path: bool = False,
588 allow_dash: bool = False,
589 path_type: type | None = None,
590 ):
591 self.mustNotExist = exists is False
592 if exists is None: 592 ↛ 594line 592 didn't jump to line 594, because the condition on line 592 was never false
593 exists = False
594 super().__init__(
595 exists=exists,
596 file_okay=file_okay,
597 dir_okay=dir_okay,
598 writable=writable,
599 readable=readable,
600 resolve_path=resolve_path,
601 allow_dash=allow_dash,
602 path_type=path_type,
603 )
605 def convert(self, value: str, param: click.Parameter | None, ctx: click.Context | None) -> Any:
606 """Called by click.ParamType to "convert values through types".
607 `click.Path` uses this step to verify Path conditions."""
608 if self.mustNotExist and os.path.exists(value):
609 self.fail(f'Path "{value}" should not exist.')
610 return super().convert(value, param, ctx)
613class MWOption(click.Option):
614 """Overrides click.Option with desired behaviors."""
616 def make_metavar(self) -> str:
617 """Overrides `click.Option.make_metavar`. Makes the metavar for the
618 help menu. Adds a space and an elipsis after the metavar name if
619 the option accepts multiple inputs, otherwise defers to the base
620 implementation.
622 By default click does not add an elipsis when multiple is True and
623 nargs is 1. And when nargs does not equal 1 click adds an elipsis
624 without a space between the metavar and the elipsis, but we prefer a
625 space between.
627 Does not get called for some option types (e.g. flag) so metavar
628 transformation that must apply to all types should be applied in
629 get_help_record.
630 """
631 metavar = super().make_metavar()
632 if self.multiple and self.nargs == 1:
633 metavar += " ..."
634 elif self.nargs != 1:
635 metavar = f"{metavar[:-3]} ..."
636 return metavar
639class MWArgument(click.Argument):
640 """Overrides click.Argument with desired behaviors."""
642 def make_metavar(self) -> str:
643 """Overrides `click.Option.make_metavar`. Makes the metavar for the
644 help menu. Always adds a space and an elipsis (' ...') after the
645 metavar name if the option accepts multiple inputs.
647 By default click adds an elipsis without a space between the metavar
648 and the elipsis, but we prefer a space between.
650 Returns
651 -------
652 metavar : `str`
653 The metavar value.
654 """
655 metavar = super().make_metavar()
656 if self.nargs != 1:
657 metavar = f"{metavar[:-3]} ..."
658 return metavar
661class OptionSection(MWOption):
662 """Implements an Option that prints a section label in the help text and
663 does not pass any value to the command function.
665 This class does a bit of hackery to add a section label to a click command
666 help output: first, `expose_value` is set to `False` so that no value is
667 passed to the command function. Second, this class overrides
668 `click.Option.get_help_record` to return the section label string without
669 any prefix so that it stands out as a section label.
671 This class overrides the hidden attribute because our documentation build
672 tool, sphinx-click, implements its own `get_help_record` function which
673 builds the record from other option values (e.g. `name`, `opts`), which
674 breaks the hack we use to make `get_help_record` only return the
675 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
676 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
677 entering its `_get_help_record` function. So, making the hidden property
678 return True hides this option from sphinx-click, while allowing the section
679 text to be returned by our `get_help_record` method when using Click.
681 The intention for this implementation is to do minimally invasive overrides
682 of the click classes so as to be robust and easy to fix if the click
683 internals change.
685 Parameters
686 ----------
687 sectionName : `str`
688 The parameter declaration for this option. It is not shown to the user,
689 it must be unique within the command. If using the `section` decorator
690 to add a section to a command's options, the section name is
691 auto-generated.
692 sectionText : `str`
693 The text to print in the section identifier.
694 """
696 @property
697 def hidden(self) -> bool:
698 return True
700 @hidden.setter
701 def hidden(self, val: Any) -> None:
702 pass
704 def __init__(self, sectionName: str, sectionText: str) -> None:
705 super().__init__(sectionName, expose_value=False)
706 self.sectionText = sectionText
708 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
709 return (self.sectionText, "")
712class MWOptionDecorator:
713 """Wraps the click.option decorator to enable shared options to be declared
714 and allows inspection of the shared option.
715 """
717 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
718 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs)
719 opt = click.Option(param_decls, **kwargs)
720 self._name = opt.name
721 self._opts = opt.opts
723 def name(self) -> str:
724 """Get the name that will be passed to the command function for this
725 option."""
726 return self._name
728 def opts(self) -> list[str]:
729 """Get the flags that will be used for this option on the command
730 line."""
731 return self._opts
733 @property
734 def help(self) -> str:
735 """Get the help text for this option. Returns an empty string if no
736 help was defined."""
737 return self.partialOpt.keywords.get("help", "")
739 def __call__(self, *args: Any, **kwargs: Any) -> Any:
740 return self.partialOpt(*args, **kwargs)
743class MWArgumentDecorator:
744 """Wraps the click.argument decorator to enable shared arguments to be
745 declared."""
747 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
748 self._helpText = kwargs.pop("help", None)
749 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
751 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
752 def decorator(f: Any) -> Any:
753 if help is not None:
754 self._helpText = help
755 if self._helpText:
756 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
757 return self.partialArg(*args, **kwargs)(f)
759 return decorator
762class MWCommand(click.Command):
763 """Command subclass that stores a copy of the args list for use by the
764 command."""
766 extra_epilog: str | None = None
768 def __init__(self, *args: Any, **kwargs: Any) -> None:
769 # wrap callback method with catch_and_exit decorator
770 callback = kwargs.get("callback")
771 if callback is not None: 771 ↛ 774line 771 didn't jump to line 774, because the condition on line 771 was never false
772 kwargs = kwargs.copy()
773 kwargs["callback"] = catch_and_exit(callback)
774 super().__init__(*args, **kwargs)
776 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
777 """Capture the command line options and arguments.
779 See details about what is captured and the order in which it is stored
780 in the documentation of `MWCtxObj`.
782 Parameters
783 ----------
784 ctx : `click.Context`
785 The current Context.
786 args : `list` [`str`]
787 The list of arguments from the command line, split at spaces but
788 not at separators (like "=").
789 """
790 parser = self.make_parser(ctx)
791 opts, _, param_order = parser.parse_args(args=list(args))
792 # `param_order` is a list of click.Option and click.Argument, there is
793 # one item for each time the Option or Argument was used on the
794 # command line. Options will precede Arguments, within each sublist
795 # they are in the order they were used on the command line. Note that
796 # click.Option and click.Argument do not contain the value from the
797 # command line; values are in `opts`.
798 #
799 # `opts` is a dict where the key is the argument name to the
800 # click.Command function, this name matches the `click.Option.name` or
801 # `click.Argument.name`. For Options, an item will only be present if
802 # the Option was used on the command line. For Arguments, an item will
803 # always be present and if no value was provided on the command line
804 # the value will be `None`. If the option accepts multiple values, the
805 # value in `opts` is a tuple, otherwise it is a single item.
806 next_idx: Counter = Counter()
807 captured_args = []
808 for param in param_order:
809 if isinstance(param, click.Option):
810 if param.multiple:
811 val = opts[param.name][next_idx[param.name]]
812 next_idx[param.name] += 1
813 else:
814 val = opts[param.name]
815 if param.is_flag:
816 # Bool options store their True flags in opts and their
817 # False flags in secondary_opts.
818 if val:
819 flag = max(param.opts, key=len)
820 else:
821 flag = max(param.secondary_opts, key=len)
822 captured_args.append(flag)
823 else:
824 captured_args.append(max(param.opts, key=len))
825 captured_args.append(val)
826 elif isinstance(param, click.Argument):
827 if (opt := opts[param.name]) is not None:
828 captured_args.append(opt)
829 else:
830 assert False # All parameters should be an Option or an Argument.
831 MWCtxObj.getFrom(ctx).args = captured_args
833 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
834 """Given a context and a list of arguments this creates the parser and
835 parses the arguments, then modifies the context as necessary. This is
836 automatically invoked by make_context().
838 This function overrides `click.Command.parse_args`.
840 The call to `_capture_args` in this override stores the arguments
841 (option names, option value, and argument values) that were used by the
842 caller on the command line in the context object. These stored
843 arugments can be used by the command function, e.g. to process options
844 in the order they appeared on the command line (pipetask uses this
845 feature to create pipeline actions in an order from different options).
847 Parameters
848 ----------
849 ctx : `click.core.Context`
850 The current Context.ß
851 args : `list` [`str`]
852 The list of arguments from the command line, split at spaces but
853 not at separators (like "=").
854 """
855 self._capture_args(ctx, args)
856 return super().parse_args(ctx, args)
858 @property
859 def epilog(self) -> str | None:
860 """Override the epilog attribute to add extra_epilog (if defined by a
861 subclass) to the end of any epilog provided by a subcommand.
862 """
863 ret = self._epilog if self._epilog else ""
864 if self.extra_epilog:
865 if ret:
866 ret += "\n\n"
867 ret += self.extra_epilog
868 return ret
870 @epilog.setter
871 def epilog(self, val: str) -> None:
872 self._epilog = val
875class ButlerCommand(MWCommand):
876 """Command subclass with butler-command specific overrides."""
878 extra_epilog = "See 'butler --help' for more options."
881class OptionGroup:
882 """Base class for an option group decorator. Requires the option group
883 subclass to have a property called `decorator`."""
885 decorators: list[Any]
887 def __call__(self, f: Any) -> Any:
888 for decorator in reversed(self.decorators):
889 f = decorator(f)
890 return f
893class MWCtxObj:
894 """Helper object for managing the `click.Context.obj` parameter, allows
895 obj data to be managed in a consistent way.
897 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
898 initialize the obj if needed and return a new or existing `MWCtxObj`.
900 The `args` attribute contains a list of options, option values, and
901 argument values that is similar to the list of arguments and options that
902 were passed in on the command line, with differences noted below:
904 * Option namess and option values are first in the list, and argument
905 values come last. The order of options and option values is preserved
906 within the options. The order of argument values is preserved.
908 * The longest option name is used for the option in the `args` list, e.g.
909 if an option accepts both short and long names "-o / --option" and the
910 short option name "-o" was used on the command line, the longer name will
911 be the one that appears in `args`.
913 * A long option name (which begins with two dashes "--") and its value may
914 be separated by an equal sign; the name and value are split at the equal
915 sign and it is removed. In `args`, the option is in one list item, and
916 the option value (without the equal sign) is in the next list item. e.g.
917 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in
918 `args`.
920 * A short option name, (which begins with one dash "-") and its value are
921 split immediately after the short option name, and if there is
922 whitespace between the short option name and its value it is removed.
923 Everything after the short option name (excluding whitespace) is included
924 in the value. If the `Option` has a long name, the long name will be used
925 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo"
926 become `["--option", "foo"]`, and (note!) "-o=foo" will become
927 `["--option", "=foo"]` (because everything after the short option name,
928 except whitespace, is used for the value (as is standard with unix
929 command line tools).
931 Attributes
932 ----------
933 args : `list` [`str`]
934 A list of options, option values, and arguments simialr to those that
935 were passed in on the command line. See comments about captured options
936 & arguments above.
937 """
939 def __init__(self) -> None:
940 self.args = None
942 @staticmethod
943 def getFrom(ctx: click.Context) -> Any:
944 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return
945 the new or already existing `MWCtxObj`."""
946 if ctx.obj is not None:
947 return ctx.obj
948 ctx.obj = MWCtxObj()
949 return ctx.obj
952def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
953 """Click callback that reads additional values from the supplied
954 YAML file.
956 Parameters
957 ----------
958 ctx : `click.context`
959 The context for the click operation. Used to extract the subcommand
960 name and translate option & argument names.
961 param : `str`
962 The parameter name.
963 value : `object`
964 The value of the parameter.
965 """
967 def _name_for_option(ctx: click.Context, option: str) -> str:
968 """Use a CLI option name to find the name of the argument to the
969 command function.
971 Parameters
972 ----------
973 ctx : `click.Context`
974 The context for the click operation.
975 option : `str`
976 The option/argument name from the yaml file.
978 Returns
979 -------
980 name : str
981 The name of the argument to use when calling the click.command
982 function, as it should appear in the `ctx.default_map`.
984 Raises
985 ------
986 RuntimeError
987 Raised if the option name from the yaml file does not exist in the
988 command parameters. This catches misspellings and incorrect useage
989 in the yaml file.
990 """
991 for param in ctx.command.params:
992 # Remove leading dashes: they are not used for option names in the
993 # yaml file.
994 if option in [opt.lstrip("-") for opt in param.opts]:
995 return param.name
996 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
998 ctx.default_map = ctx.default_map or {}
999 cmd_name = ctx.info_name
1000 if value:
1001 try:
1002 overrides = _read_yaml_presets(value, cmd_name)
1003 options = list(overrides.keys())
1004 for option in options:
1005 name = _name_for_option(ctx, option)
1006 if name == option:
1007 continue
1008 overrides[name] = overrides.pop(option)
1009 except Exception as e:
1010 raise click.BadOptionUsage(
1011 option_name=param,
1012 message=f"Error reading overrides file: {e}",
1013 ctx=ctx,
1014 )
1015 # Override the defaults for this subcommand
1016 # mypy: (this is declared Mapping not MutableMapping so must be
1017 # ignored)
1018 ctx.default_map.update(overrides) # type: ignore[attr-defined]
1019 return
1022def _read_yaml_presets(file_uri: str, cmd_name: str | None) -> dict[str, Any]:
1023 """Read file command line overrides from YAML config file.
1025 Parameters
1026 ----------
1027 file_uri : `str`
1028 URI of override YAML file containing the command line overrides.
1029 They should be grouped by command name.
1030 cmd_name : `str`
1031 The subcommand name that is being modified.
1033 Returns
1034 -------
1035 overrides : `dict` of [`str`, Any]
1036 The relevant command line options read from the override file.
1037 """
1038 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1039 config = Config(file_uri)
1040 return config[cmd_name]
1043def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1044 """Sort an astropy table, with prioritization given to columns in this
1045 order:
1046 1. the provided named columns
1047 2. spatial and temporal columns
1048 3. the rest of the columns
1050 The table is sorted in-place, and is also returned for convenience.
1052 Parameters
1053 ----------
1054 table : `astropy.table.Table`
1055 The table to sort
1056 dimensions : `list` [``Dimension``]
1057 The dimensions of the dataIds in the table (the dimensions should be
1058 the same for all the dataIds). Used to determine if the column is
1059 spatial, temporal, or neither.
1060 sort_first : `list` [`str`]
1061 The names of columns that should be sorted first, before spatial and
1062 temporal columns.
1064 Returns
1065 -------
1066 `astropy.table.Table`
1067 For convenience, the table that has been sorted.
1068 """
1069 # For sorting we want to ignore the id
1070 # We also want to move temporal or spatial dimensions earlier
1071 sort_first = sort_first or []
1072 sort_early: list[str] = []
1073 sort_late: list[str] = []
1074 for dim in dimensions:
1075 if dim.spatial or dim.temporal:
1076 sort_early.extend(dim.required.names)
1077 else:
1078 sort_late.append(str(dim))
1079 sort_keys = sort_first + sort_early + sort_late
1080 # The required names above means that we have the possibility of
1081 # repeats of sort keys. Now have to remove them
1082 # (order is retained by dict creation).
1083 sort_keys = list(dict.fromkeys(sort_keys).keys())
1085 table.sort(sort_keys)
1086 return table
1089def catch_and_exit(func: Callable) -> Callable:
1090 """Decorator which catches all exceptions, prints an exception traceback
1091 and signals click to exit.
1092 """
1094 @wraps(func)
1095 def inner(*args: Any, **kwargs: Any) -> None:
1096 try:
1097 func(*args, **kwargs)
1098 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1099 # this is handled by click itself
1100 raise
1101 except Exception:
1102 exc_type, exc_value, exc_tb = sys.exc_info()
1103 assert exc_type is not None
1104 assert exc_value is not None
1105 assert exc_tb is not None
1106 if exc_tb.tb_next:
1107 # do not show this decorator in traceback
1108 exc_tb = exc_tb.tb_next
1109 log.exception(
1110 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb)
1111 )
1112 # tell click to stop, this never returns.
1113 click.get_current_context().exit(1)
1115 return inner