Coverage for python/lsst/daf/butler/cli/utils.py: 32%
353 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-28 10:10 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21from __future__ import annotations
23__all__ = (
24 "astropyTablesToStr",
25 "printAstropyTables",
26 "textTypeStr",
27 "LogCliRunner",
28 "clickResultMsg",
29 "command_test_env",
30 "addArgumentHelp",
31 "split_commas",
32 "split_kv",
33 "to_upper",
34 "unwrap",
35 "option_section",
36 "MWPath",
37 "MWOption",
38 "MWArgument",
39 "OptionSection",
40 "MWOptionDecorator",
41 "MWArgumentDecorator",
42 "MWCommand",
43 "ButlerCommand",
44 "OptionGroup",
45 "MWCtxObj",
46 "yaml_presets",
47 "sortAstropyTable",
48 "catch_and_exit",
49)
52import itertools
53import logging
54import os
55import re
56import sys
57import textwrap
58import traceback
59import uuid
60import warnings
61from collections import Counter
62from collections.abc import Callable, Iterable, Iterator
63from contextlib import contextmanager
64from functools import partial, wraps
65from typing import TYPE_CHECKING, Any, cast
66from unittest.mock import patch
68import click
69import click.exceptions
70import click.testing
71import yaml
72from lsst.utils.iteration import ensure_iterable
74from ..core.config import Config
75from .cliLog import CliLog
77if TYPE_CHECKING:
78 from astropy.table import Table
79 from lsst.daf.butler import Dimension
81log = logging.getLogger(__name__)
83# This is used as the metavar argument to Options that accept multiple string
84# inputs, which may be comma-separarated. For example:
85# --my-opt foo,bar --my-opt baz.
86# Other arguments to the Option should include multiple=true and
87# callback=split_kv.
88typeStrAcceptsMultiple = "TEXT ..."
89typeStrAcceptsSingle = "TEXT"
91# The standard help string for the --where option when it takes a WHERE clause.
92where_help = (
93 "A string expression similar to a SQL WHERE clause. May involve any column of a "
94 "dimension table or a dimension name as a shortcut for the primary key column of a "
95 "dimension table."
96)
99def astropyTablesToStr(tables: list[Table]) -> str:
100 """Render astropy tables to string as they are displayed in the CLI.
102 Output formatting matches ``printAstropyTables``.
103 """
104 ret = ""
105 for table in tables:
106 ret += "\n"
107 table.pformat_all()
108 ret += "\n"
109 return ret
112def printAstropyTables(tables: list[Table]) -> None:
113 """Print astropy tables to be displayed in the CLI.
115 Output formatting matches ``astropyTablesToStr``.
116 """
117 for table in tables:
118 print("")
119 table.pprint_all()
120 print("")
123def textTypeStr(multiple: bool) -> str:
124 """Get the text type string for CLI help documentation.
126 Parameters
127 ----------
128 multiple : `bool`
129 True if multiple text values are allowed, False if only one value is
130 allowed.
132 Returns
133 -------
134 textTypeStr : `str`
135 The type string to use.
136 """
137 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
140class LogCliRunner(click.testing.CliRunner):
141 """A test runner to use when the logging system will be initialized by code
142 under test, calls CliLog.resetLog(), which undoes any logging setup that
143 was done with the CliLog interface.
145 lsst.log modules can not be set back to an uninitialized state (python
146 logging modules can be set back to NOTSET), instead they are set to
147 `CliLog.defaultLsstLogLevel`.
148 """
150 def invoke(self, *args: Any, **kwargs: Any) -> Any:
151 result = super().invoke(*args, **kwargs)
152 CliLog.resetLog()
153 return result
156def clickResultMsg(result: click.testing.Result) -> str:
157 """Get a standard assert message from a click result.
159 Parameters
160 ----------
161 result : click.testing.Result
162 The result object returned from click.testing.CliRunner.invoke
164 Returns
165 -------
166 msg : `str`
167 The message string.
168 """
169 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
170 if result.exception:
171 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
172 return msg
175@contextmanager
176def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
177 """Context manager that creates (and then cleans up) an environment that
178 provides a CLI plugin command with the given name.
180 Parameters
181 ----------
182 runner : click.testing.CliRunner
183 The test runner to use to create the isolated filesystem.
184 commandModule : `str`
185 The importable module that the command can be imported from.
186 commandName : `str`
187 The name of the command being published to import.
188 """
189 with runner.isolated_filesystem():
190 with open("resources.yaml", "w") as f:
191 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
192 # Add a colon to the end of the path on the next line, this tests the
193 # case where the lookup in LoaderCLI._getPluginList generates an empty
194 # string in one of the list entries and verifies that the empty string
195 # is properly stripped out.
196 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
197 yield
200def addArgumentHelp(doc: str | None, helpText: str) -> str:
201 """Add a Click argument's help message to a function's documentation.
203 This is needed because click presents arguments in the order the argument
204 decorators are applied to a function, top down. But, the evaluation of the
205 decorators happens bottom up, so if arguments just append their help to the
206 function's docstring, the argument descriptions appear in reverse order
207 from the order they are applied in.
209 Parameters
210 ----------
211 doc : `str`
212 The function's docstring.
213 helpText : `str`
214 The argument's help string to be inserted into the function's
215 docstring.
217 Returns
218 -------
219 doc : `str`
220 Updated function documentation.
221 """
222 if doc is None: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true
223 doc = helpText
224 else:
225 # See click documentation for details:
226 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
227 # In short, text for the click command help can be truncated by putting
228 # "\f" in the docstring, everything after it should be removed
229 if "\f" in doc: 229 ↛ 230line 229 didn't jump to line 230, because the condition on line 229 was never true
230 doc = doc.split("\f")[0]
232 doclines = doc.splitlines()
233 # The function's docstring may span multiple lines, so combine the
234 # docstring from all the first lines until a blank line is encountered.
235 # (Lines after the first blank line will be argument help.)
236 while len(doclines) > 1 and doclines[1]:
237 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
238 # Add standard indent to help text for proper alignment with command
239 # function documentation:
240 helpText = " " + helpText
241 doclines.insert(1, helpText)
242 doclines.insert(1, "\n")
243 doc = "\n".join(doclines)
244 return doc
247def split_commas(
248 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None
249) -> tuple[str, ...]:
250 """Process a tuple of values, where each value may contain comma-separated
251 values, and return a single list of all the passed-in values.
253 This function can be passed to the 'callback' argument of a click.option to
254 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
255 the comma is inside ``[]`` there will be no splitting.
257 Parameters
258 ----------
259 context : `click.Context` or `None`
260 The current execution context. Unused, but Click always passes it to
261 callbacks.
262 param : `click.core.Option` or `None`
263 The parameter being handled. Unused, but Click always passes it to
264 callbacks.
265 values : iterable of `str` or `str`
266 All the values passed for this option. Strings may contain commas,
267 which will be treated as delimiters for separate values unless they
268 are within ``[]``.
270 Returns
271 -------
272 results : `tuple` [`str`]
273 The passed in values separated by commas where appropriate and
274 combined into a single tuple.
275 """
276 if values is None:
277 return tuple()
278 valueList = []
279 for value in ensure_iterable(values):
280 # If we have [, or ,] we do the slow split. If square brackets
281 # are not matching then that is likely a typo that should result
282 # in a warning.
283 opens = "["
284 closes = "]"
285 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
286 in_parens = False
287 current = ""
288 for c in value:
289 if c == opens:
290 if in_parens:
291 warnings.warn(
292 f"Found second opening {opens} without corresponding closing {closes}"
293 f" in {value!r}",
294 stacklevel=2,
295 )
296 in_parens = True
297 elif c == closes:
298 if not in_parens:
299 warnings.warn(
300 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
301 stacklevel=2,
302 )
303 in_parens = False
304 elif c == ",":
305 if not in_parens:
306 # Split on this comma.
307 valueList.append(current)
308 current = ""
309 continue
310 current += c
311 if in_parens:
312 warnings.warn(
313 f"Found opening {opens} that was never closed in {value!r}",
314 stacklevel=2,
315 )
316 if current:
317 valueList.append(current)
318 else:
319 # Use efficient split since no parens.
320 valueList.extend(value.split(","))
321 return tuple(valueList)
324def split_kv(
325 context: click.Context,
326 param: click.core.Option,
327 values: list[str],
328 *,
329 choice: click.Choice | None = None,
330 multiple: bool = True,
331 normalize: bool = False,
332 separator: str = "=",
333 unseparated_okay: bool = False,
334 return_type: type[dict] | type[tuple] = dict,
335 default_key: str = "",
336 reverse_kv: bool = False,
337 add_to_default: bool = False,
338) -> dict[str, str] | tuple[tuple[str, str], ...]:
339 """Process a tuple of values that are key-value pairs separated by a given
340 separator. Multiple pairs may be comma separated. Return a dictionary of
341 all the passed-in values.
343 This function can be passed to the 'callback' argument of a click.option to
344 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
346 Parameters
347 ----------
348 context : `click.Context` or `None`
349 The current execution context. Unused, but Click always passes it to
350 callbacks.
351 param : `click.core.Option` or `None`
352 The parameter being handled. Unused, but Click always passes it to
353 callbacks.
354 values : [`str`]
355 All the values passed for this option. Strings may contain commas,
356 which will be treated as delimiters for separate values.
357 choice : `click.Choice`, optional
358 If provided, verify each value is a valid choice using the provided
359 `click.Choice` instance. If None, no verification will be done. By
360 default None
361 multiple : `bool`, optional
362 If true, the value may contain multiple comma-separated values. By
363 default True.
364 normalize : `bool`, optional
365 If True and `choice.case_sensitive == False`, normalize the string the
366 user provided to match the choice's case. By default False.
367 separator : str, optional
368 The character that separates key-value pairs. May not be a comma or an
369 empty space (for space separators use Click's default implementation
370 for tuples; `type=(str, str)`). By default "=".
371 unseparated_okay : `bool`, optional
372 If True, allow values that do not have a separator. They will be
373 returned in the values dict as a tuple of values in the key '', that
374 is: `values[''] = (unseparated_values, )`. By default False.
375 return_type : `type`, must be `dict` or `tuple`
376 The type of the value that should be returned.
377 If `dict` then the returned object will be a dict, for each item in
378 values, the value to the left of the separator will be the key and the
379 value to the right of the separator will be the value.
380 If `tuple` then the returned object will be a tuple. Each item in the
381 tuple will be 2-item tuple, the first item will be the value to the
382 left of the separator and the second item will be the value to the
383 right. By default `dict`.
384 default_key : `Any`
385 The key to use if a value is passed that is not a key-value pair.
386 (Passing values that are not key-value pairs requires
387 ``unseparated_okay`` to be `True`.)
388 reverse_kv : bool
389 If true then for each item in values, the value to the left of the
390 separator is treated as the value and the value to the right of the
391 separator is treated as the key. By default False.
392 add_to_default : `bool`, optional
393 If True, then passed-in values will not overwrite the default value
394 unless the ``return_type`` is `dict` and passed-in value(s) have the
395 same key(s) as the default value.
397 Returns
398 -------
399 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...]
400 The passed-in values in dict form or tuple form.
402 Raises
403 ------
404 `click.ClickException`
405 Raised if the separator is not found in an entry, or if duplicate keys
406 are encountered.
407 """
409 def norm(val: str) -> str:
410 """If `normalize` is True and `choice` is not `None`, find the value
411 in the available choices and return the value as spelled in the
412 choices.
414 Assumes that val exists in choices; `split_kv` uses the `choice`
415 instance to verify val is a valid choice.
416 """
417 if normalize and choice is not None:
418 v = val.casefold()
419 for opt in choice.choices:
420 if opt.casefold() == v:
421 return opt
422 return val
424 class RetDict:
425 def __init__(self) -> None:
426 self.ret: dict[str, str] = {}
428 def add(self, key: str, val: str) -> None:
429 if reverse_kv:
430 key, val = val, key
431 self.ret[key] = val
433 def get(self) -> dict[str, str]:
434 return self.ret
436 class RetTuple:
437 def __init__(self) -> None:
438 self.ret: list[tuple[str, str]] = []
440 def add(self, key: str, val: str) -> None:
441 if reverse_kv:
442 key, val = val, key
443 self.ret.append((key, val))
445 def get(self) -> tuple[tuple[str, str], ...]:
446 return tuple(self.ret)
448 if separator in (",", " "):
449 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
450 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
452 if add_to_default:
453 default = param.get_default(context)
454 if default:
455 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy
457 ret: RetDict | RetTuple
458 if return_type is dict:
459 ret = RetDict()
460 elif return_type is tuple:
461 ret = RetTuple()
462 else:
463 raise click.ClickException(
464 message=f"Internal error: invalid return type '{return_type}' for split_kv."
465 )
466 if multiple:
467 vals = split_commas(context, param, vals)
468 for val in ensure_iterable(vals):
469 if unseparated_okay and separator not in val:
470 if choice is not None:
471 choice(val) # will raise if val is an invalid choice
472 ret.add(default_key, norm(val))
473 else:
474 try:
475 k, v = val.split(separator)
476 if choice is not None:
477 choice(v) # will raise if val is an invalid choice
478 except ValueError as e:
479 raise click.ClickException(
480 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
481 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
482 )
483 ret.add(k, norm(v))
484 return ret.get()
487def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
488 """Convert a value to upper case.
490 Parameters
491 ----------
492 context : click.Context
494 values : string
495 The value to be converted.
497 Returns
498 -------
499 string
500 A copy of the passed-in value, converted to upper case.
501 """
502 return value.upper()
505def unwrap(val: str) -> str:
506 """Remove newlines and leading whitespace from a multi-line string with
507 a consistent indentation level.
509 The first line of the string may be only a newline or may contain text
510 followed by a newline, either is ok. After the first line, each line must
511 begin with a consistant amount of whitespace. So, content of a
512 triple-quoted string may begin immediately after the quotes, or the string
513 may start with a newline. Each line after that must be the same amount of
514 indentation/whitespace followed by text and a newline. The last line may
515 end with a new line but is not required to do so.
517 Parameters
518 ----------
519 val : `str`
520 The string to change.
522 Returns
523 -------
524 strippedString : `str`
525 The string with newlines, indentation, and leading and trailing
526 whitespace removed.
527 """
529 def splitSection(val: str) -> str:
530 if not val.startswith("\n"): 530 ↛ 534line 530 didn't jump to line 534, because the condition on line 530 was never false
531 firstLine, _, val = val.partition("\n")
532 firstLine += " "
533 else:
534 firstLine = ""
535 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
537 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
540class option_section: # noqa: N801
541 """Decorator to add a section label between options in the help text of a
542 command.
544 Parameters
545 ----------
546 sectionText : `str`
547 The text to print in the section identifier.
548 """
550 def __init__(self, sectionText: str) -> None:
551 self.sectionText = "\n" + sectionText
553 def __call__(self, f: Any) -> click.Option:
554 # Generate a parameter declaration that will be unique for this
555 # section.
556 return click.option(
557 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
558 )(f)
561class MWPath(click.Path):
562 """Overrides click.Path to implement file-does-not-exist checking.
564 Changes the definition of ``exists` so that `True` indicates the location
565 (file or directory) must exist, `False` indicates the location must *not*
566 exist, and `None` indicates that the file may exist or not. The standard
567 definition for the `click.Path` ``exists`` parameter is that for `True` a
568 location must exist, but `False` means it is not required to exist (not
569 that it is required to not exist).
571 Parameters
572 ----------
573 exists : `True`, `False`, or `None`
574 If `True`, the location (file or directory) indicated by the caller
575 must exist. If `False` the location must not exist. If `None`, the
576 location may exist or not.
578 For other parameters see `click.Path`.
579 """
581 def __init__(
582 self,
583 exists: bool | None = None,
584 file_okay: bool = True,
585 dir_okay: bool = True,
586 writable: bool = False,
587 readable: bool = True,
588 resolve_path: bool = False,
589 allow_dash: bool = False,
590 path_type: type | None = None,
591 ):
592 self.mustNotExist = exists is False
593 if exists is None: 593 ↛ 595line 593 didn't jump to line 595, because the condition on line 593 was never false
594 exists = False
595 super().__init__(
596 exists=exists,
597 file_okay=file_okay,
598 dir_okay=dir_okay,
599 writable=writable,
600 readable=readable,
601 resolve_path=resolve_path,
602 allow_dash=allow_dash,
603 path_type=path_type,
604 )
606 def convert(self, value: str, param: click.Parameter | None, ctx: click.Context | None) -> Any:
607 """Convert values through types.
609 Called by `click.ParamType` to "convert values through types".
610 `click.Path` uses this step to verify Path conditions.
611 """
612 if self.mustNotExist and os.path.exists(value):
613 self.fail(f'Path "{value}" should not exist.')
614 return super().convert(value, param, ctx)
617class MWOption(click.Option):
618 """Overrides click.Option with desired behaviors."""
620 def make_metavar(self) -> str:
621 """Make the metavar for the help menu.
623 Overrides `click.Option.make_metavar`.
624 Adds a space and an ellipsis after the metavar name if
625 the option accepts multiple inputs, otherwise defers to the base
626 implementation.
628 By default click does not add an ellipsis when multiple is True and
629 nargs is 1. And when nargs does not equal 1 click adds an ellipsis
630 without a space between the metavar and the ellipsis, but we prefer a
631 space between.
633 Does not get called for some option types (e.g. flag) so metavar
634 transformation that must apply to all types should be applied in
635 get_help_record.
636 """
637 metavar = super().make_metavar()
638 if self.multiple and self.nargs == 1:
639 metavar += " ..."
640 elif self.nargs != 1:
641 metavar = f"{metavar[:-3]} ..."
642 return metavar
645class MWArgument(click.Argument):
646 """Overrides click.Argument with desired behaviors."""
648 def make_metavar(self) -> str:
649 """Make the metavar for the help menu.
651 Overrides `click.Option.make_metavar`.
652 Always adds a space and an ellipsis (' ...') after the
653 metavar name if the option accepts multiple inputs.
655 By default click adds an ellipsis without a space between the metavar
656 and the ellipsis, but we prefer a space between.
658 Returns
659 -------
660 metavar : `str`
661 The metavar value.
662 """
663 metavar = super().make_metavar()
664 if self.nargs != 1:
665 metavar = f"{metavar[:-3]} ..."
666 return metavar
669class OptionSection(MWOption):
670 """Implements an Option that prints a section label in the help text and
671 does not pass any value to the command function.
673 This class does a bit of hackery to add a section label to a click command
674 help output: first, `expose_value` is set to `False` so that no value is
675 passed to the command function. Second, this class overrides
676 `click.Option.get_help_record` to return the section label string without
677 any prefix so that it stands out as a section label.
679 This class overrides the hidden attribute because our documentation build
680 tool, sphinx-click, implements its own `get_help_record` function which
681 builds the record from other option values (e.g. `name`, `opts`), which
682 breaks the hack we use to make `get_help_record` only return the
683 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
684 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
685 entering its `_get_help_record` function. So, making the hidden property
686 return True hides this option from sphinx-click, while allowing the section
687 text to be returned by our `get_help_record` method when using Click.
689 The intention for this implementation is to do minimally invasive overrides
690 of the click classes so as to be robust and easy to fix if the click
691 internals change.
693 Parameters
694 ----------
695 sectionName : `str`
696 The parameter declaration for this option. It is not shown to the user,
697 it must be unique within the command. If using the `section` decorator
698 to add a section to a command's options, the section name is
699 auto-generated.
700 sectionText : `str`
701 The text to print in the section identifier.
702 """
704 @property
705 def hidden(self) -> bool:
706 return True
708 @hidden.setter
709 def hidden(self, val: Any) -> None:
710 pass
712 def __init__(self, sectionName: str, sectionText: str) -> None:
713 super().__init__(sectionName, expose_value=False)
714 self.sectionText = sectionText
716 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
717 return (self.sectionText, "")
720class MWOptionDecorator:
721 """Wraps the click.option decorator to enable shared options to be declared
722 and allows inspection of the shared option.
723 """
725 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
726 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs)
727 opt = click.Option(param_decls, **kwargs)
728 self._name = opt.name
729 self._opts = opt.opts
731 def name(self) -> str:
732 """Get the name that will be passed to the command function for this
733 option.
734 """
735 return cast(str, self._name)
737 def opts(self) -> list[str]:
738 """Get the flags that will be used for this option on the command
739 line.
740 """
741 return self._opts
743 @property
744 def help(self) -> str:
745 """Get the help text for this option. Returns an empty string if no
746 help was defined.
747 """
748 return self.partialOpt.keywords.get("help", "")
750 def __call__(self, *args: Any, **kwargs: Any) -> Any:
751 return self.partialOpt(*args, **kwargs)
754class MWArgumentDecorator:
755 """Wraps the click.argument decorator to enable shared arguments to be
756 declared.
757 """
759 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
760 self._helpText = kwargs.pop("help", None)
761 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
763 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
764 def decorator(f: Any) -> Any:
765 if help is not None:
766 self._helpText = help
767 if self._helpText:
768 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
769 return self.partialArg(*args, **kwargs)(f)
771 return decorator
774class MWCommand(click.Command):
775 """Command subclass that stores a copy of the args list for use by the
776 command.
777 """
779 extra_epilog: str | None = None
781 def __init__(self, *args: Any, **kwargs: Any) -> None:
782 # wrap callback method with catch_and_exit decorator
783 callback = kwargs.get("callback")
784 if callback is not None: 784 ↛ 787line 784 didn't jump to line 787, because the condition on line 784 was never false
785 kwargs = kwargs.copy()
786 kwargs["callback"] = catch_and_exit(callback)
787 super().__init__(*args, **kwargs)
789 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
790 """Capture the command line options and arguments.
792 See details about what is captured and the order in which it is stored
793 in the documentation of `MWCtxObj`.
795 Parameters
796 ----------
797 ctx : `click.Context`
798 The current Context.
799 args : `list` [`str`]
800 The list of arguments from the command line, split at spaces but
801 not at separators (like "=").
802 """
803 parser = self.make_parser(ctx)
804 opts, _, param_order = parser.parse_args(args=list(args))
805 # `param_order` is a list of click.Option and click.Argument, there is
806 # one item for each time the Option or Argument was used on the
807 # command line. Options will precede Arguments, within each sublist
808 # they are in the order they were used on the command line. Note that
809 # click.Option and click.Argument do not contain the value from the
810 # command line; values are in `opts`.
811 #
812 # `opts` is a dict where the key is the argument name to the
813 # click.Command function, this name matches the `click.Option.name` or
814 # `click.Argument.name`. For Options, an item will only be present if
815 # the Option was used on the command line. For Arguments, an item will
816 # always be present and if no value was provided on the command line
817 # the value will be `None`. If the option accepts multiple values, the
818 # value in `opts` is a tuple, otherwise it is a single item.
819 next_idx: Counter = Counter()
820 captured_args = []
821 for param in param_order:
822 if isinstance(param, click.Option):
823 param_name = cast(str, param.name)
824 if param.multiple:
825 val = opts[param_name][next_idx[param_name]]
826 next_idx[param_name] += 1
827 else:
828 val = opts[param_name]
829 if param.is_flag:
830 # Bool options store their True flags in opts and their
831 # False flags in secondary_opts.
832 if val:
833 flag = max(param.opts, key=len)
834 else:
835 flag = max(param.secondary_opts, key=len)
836 captured_args.append(flag)
837 else:
838 captured_args.append(max(param.opts, key=len))
839 captured_args.append(val)
840 elif isinstance(param, click.Argument):
841 param_name = cast(str, param.name)
842 if (opt := opts[param_name]) is not None:
843 captured_args.append(opt)
844 else:
845 assert False # All parameters should be an Option or an Argument.
846 MWCtxObj.getFrom(ctx).args = captured_args
848 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
849 """Given a context and a list of arguments this creates the parser and
850 parses the arguments, then modifies the context as necessary. This is
851 automatically invoked by make_context().
853 This function overrides `click.Command.parse_args`.
855 The call to `_capture_args` in this override stores the arguments
856 (option names, option value, and argument values) that were used by the
857 caller on the command line in the context object. These stored
858 arugments can be used by the command function, e.g. to process options
859 in the order they appeared on the command line (pipetask uses this
860 feature to create pipeline actions in an order from different options).
862 Parameters
863 ----------
864 ctx : `click.core.Context`
865 The current Context.ß
866 args : `list` [`str`]
867 The list of arguments from the command line, split at spaces but
868 not at separators (like "=").
869 """
870 self._capture_args(ctx, args)
871 return super().parse_args(ctx, args)
873 @property
874 def epilog(self) -> str | None:
875 """Override the epilog attribute to add extra_epilog (if defined by a
876 subclass) to the end of any epilog provided by a subcommand.
877 """
878 ret = self._epilog if self._epilog else ""
879 if self.extra_epilog:
880 if ret:
881 ret += "\n\n"
882 ret += self.extra_epilog
883 return ret
885 @epilog.setter
886 def epilog(self, val: str) -> None:
887 self._epilog = val
890class ButlerCommand(MWCommand):
891 """Command subclass with butler-command specific overrides."""
893 extra_epilog = "See 'butler --help' for more options."
896class OptionGroup:
897 """Base class for an option group decorator. Requires the option group
898 subclass to have a property called `decorator`.
899 """
901 decorators: list[Any]
903 def __call__(self, f: Any) -> Any:
904 for decorator in reversed(self.decorators):
905 f = decorator(f)
906 return f
909class MWCtxObj:
910 """Helper object for managing the `click.Context.obj` parameter, allows
911 obj data to be managed in a consistent way.
913 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
914 initialize the obj if needed and return a new or existing `MWCtxObj`.
916 The `args` attribute contains a list of options, option values, and
917 argument values that is similar to the list of arguments and options that
918 were passed in on the command line, with differences noted below:
920 * Option namess and option values are first in the list, and argument
921 values come last. The order of options and option values is preserved
922 within the options. The order of argument values is preserved.
924 * The longest option name is used for the option in the `args` list, e.g.
925 if an option accepts both short and long names "-o / --option" and the
926 short option name "-o" was used on the command line, the longer name will
927 be the one that appears in `args`.
929 * A long option name (which begins with two dashes "--") and its value may
930 be separated by an equal sign; the name and value are split at the equal
931 sign and it is removed. In `args`, the option is in one list item, and
932 the option value (without the equal sign) is in the next list item. e.g.
933 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in
934 `args`.
936 * A short option name, (which begins with one dash "-") and its value are
937 split immediately after the short option name, and if there is
938 whitespace between the short option name and its value it is removed.
939 Everything after the short option name (excluding whitespace) is included
940 in the value. If the `Option` has a long name, the long name will be used
941 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo"
942 become `["--option", "foo"]`, and (note!) "-o=foo" will become
943 `["--option", "=foo"]` (because everything after the short option name,
944 except whitespace, is used for the value (as is standard with unix
945 command line tools).
947 Attributes
948 ----------
949 args : `list` [`str`]
950 A list of options, option values, and arguments simialr to those that
951 were passed in on the command line. See comments about captured options
952 & arguments above.
953 """
955 def __init__(self) -> None:
956 self.args = None
958 @staticmethod
959 def getFrom(ctx: click.Context) -> Any:
960 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return
961 the new or already existing `MWCtxObj`.
962 """
963 if ctx.obj is not None:
964 return ctx.obj
965 ctx.obj = MWCtxObj()
966 return ctx.obj
969def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
970 """Read additional values from the supplied YAML file.
972 Parameters
973 ----------
974 ctx : `click.context`
975 The context for the click operation. Used to extract the subcommand
976 name and translate option & argument names.
977 param : `str`
978 The parameter name.
979 value : `object`
980 The value of the parameter.
981 """
983 def _name_for_option(ctx: click.Context, option: str) -> str:
984 """Use a CLI option name to find the name of the argument to the
985 command function.
987 Parameters
988 ----------
989 ctx : `click.Context`
990 The context for the click operation.
991 option : `str`
992 The option/argument name from the yaml file.
994 Returns
995 -------
996 name : str
997 The name of the argument to use when calling the click.command
998 function, as it should appear in the `ctx.default_map`.
1000 Raises
1001 ------
1002 RuntimeError
1003 Raised if the option name from the yaml file does not exist in the
1004 command parameters. This catches misspellings and incorrect useage
1005 in the yaml file.
1006 """
1007 for param in ctx.command.params:
1008 # Remove leading dashes: they are not used for option names in the
1009 # yaml file.
1010 if option in [opt.lstrip("-") for opt in param.opts]:
1011 return cast(str, param.name)
1012 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
1014 ctx.default_map = ctx.default_map or {}
1015 cmd_name = ctx.info_name
1016 assert cmd_name is not None, "command name cannot be None"
1017 if value:
1018 try:
1019 overrides = _read_yaml_presets(value, cmd_name)
1020 options = list(overrides.keys())
1021 for option in options:
1022 name = _name_for_option(ctx, option)
1023 if name == option:
1024 continue
1025 overrides[name] = overrides.pop(option)
1026 except Exception as e:
1027 raise click.BadOptionUsage(
1028 option_name=param,
1029 message=f"Error reading overrides file: {e}",
1030 ctx=ctx,
1031 )
1032 # Override the defaults for this subcommand
1033 ctx.default_map.update(overrides)
1034 return
1037def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]:
1038 """Read file command line overrides from YAML config file.
1040 Parameters
1041 ----------
1042 file_uri : `str`
1043 URI of override YAML file containing the command line overrides.
1044 They should be grouped by command name.
1045 cmd_name : `str`
1046 The subcommand name that is being modified.
1048 Returns
1049 -------
1050 overrides : `dict` of [`str`, Any]
1051 The relevant command line options read from the override file.
1052 """
1053 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1054 config = Config(file_uri)
1055 return config[cmd_name]
1058def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1059 """Sort an astropy table, with prioritization given to columns in this
1060 order:
1061 1. the provided named columns
1062 2. spatial and temporal columns
1063 3. the rest of the columns.
1065 The table is sorted in-place, and is also returned for convenience.
1067 Parameters
1068 ----------
1069 table : `astropy.table.Table`
1070 The table to sort
1071 dimensions : `list` [``Dimension``]
1072 The dimensions of the dataIds in the table (the dimensions should be
1073 the same for all the dataIds). Used to determine if the column is
1074 spatial, temporal, or neither.
1075 sort_first : `list` [`str`]
1076 The names of columns that should be sorted first, before spatial and
1077 temporal columns.
1079 Returns
1080 -------
1081 `astropy.table.Table`
1082 For convenience, the table that has been sorted.
1083 """
1084 # For sorting we want to ignore the id
1085 # We also want to move temporal or spatial dimensions earlier
1086 sort_first = sort_first or []
1087 sort_early: list[str] = []
1088 sort_late: list[str] = []
1089 for dim in dimensions:
1090 if dim.spatial or dim.temporal:
1091 sort_early.extend(dim.required.names)
1092 else:
1093 sort_late.append(str(dim))
1094 sort_keys = sort_first + sort_early + sort_late
1095 # The required names above means that we have the possibility of
1096 # repeats of sort keys. Now have to remove them
1097 # (order is retained by dict creation).
1098 sort_keys = list(dict.fromkeys(sort_keys).keys())
1100 table.sort(sort_keys)
1101 return table
1104def catch_and_exit(func: Callable) -> Callable:
1105 """Catch all exceptions, prints an exception traceback
1106 and signals click to exit.
1108 Use as decorator.
1109 """
1111 @wraps(func)
1112 def inner(*args: Any, **kwargs: Any) -> None:
1113 try:
1114 func(*args, **kwargs)
1115 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1116 # this is handled by click itself
1117 raise
1118 except Exception:
1119 exc_type, exc_value, exc_tb = sys.exc_info()
1120 assert exc_type is not None
1121 assert exc_value is not None
1122 assert exc_tb is not None
1123 if exc_tb.tb_next:
1124 # do not show this decorator in traceback
1125 exc_tb = exc_tb.tb_next
1126 log.exception(
1127 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb)
1128 )
1129 # tell click to stop, this never returns.
1130 click.get_current_context().exit(1)
1132 return inner