Coverage for python / lsst / daf / butler / cli / utils.py: 33%
389 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-14 23:37 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "ButlerCommand",
31 "LogCliRunner",
32 "MWArgument",
33 "MWArgumentDecorator",
34 "MWCommand",
35 "MWCtxObj",
36 "MWOption",
37 "MWOptionDecorator",
38 "MWPath",
39 "OptionGroup",
40 "OptionSection",
41 "addArgumentHelp",
42 "astropyTablesToStr",
43 "catch_and_exit",
44 "clickResultMsg",
45 "command_test_env",
46 "option_section",
47 "printAstropyTables",
48 "sortAstropyTable",
49 "split_commas",
50 "split_kv",
51 "textTypeStr",
52 "to_upper",
53 "unwrap",
54 "yaml_presets",
55)
58import importlib.metadata
59import itertools
60import logging
61import os
62import re
63import sys
64import textwrap
65import traceback
66import types
67import uuid
68import warnings
69from collections import Counter
70from collections.abc import Callable, Iterable, Iterator
71from contextlib import contextmanager
72from functools import partial, wraps
73from typing import TYPE_CHECKING, Any, cast
74from unittest.mock import patch
76import click
77import click.core
78import click.exceptions
79import click.testing
80import yaml
81from packaging.version import Version
83from lsst.utils.iteration import ensure_iterable
85from .._config import Config
86from .cliLog import CliLog
88if TYPE_CHECKING:
89 from astropy.table import Table
91 from lsst.daf.butler import Dimension
93_click_version = Version(importlib.metadata.version("click"))
94if _click_version >= Version("8.2.0"): 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true
95 _click_make_metavar_has_context = True
96else:
97 _click_make_metavar_has_context = False
99# Starting from Click 8.3.0, a special `UNSET` sentinel value is used to
100# indicate the absence of a default value for a parameter. Prior to 8.3.0,
101# they just used `None`.
102_CLICK_UNSET_SENTINEL = getattr(click.core, "UNSET", None)
104log = logging.getLogger(__name__)
106# This is used as the metavar argument to Options that accept multiple string
107# inputs, which may be comma-separarated. For example:
108# --my-opt foo,bar --my-opt baz.
109# Other arguments to the Option should include multiple=true and
110# callback=split_kv.
111typeStrAcceptsMultiple = "TEXT ..."
112typeStrAcceptsSingle = "TEXT"
114# The standard help string for the --where option when it takes a WHERE clause.
115where_help = (
116 "A string expression similar to a SQL WHERE clause. May involve any column of a "
117 "dimension table or a dimension name as a shortcut for the primary key column of a "
118 "dimension table."
119)
122def astropyTablesToStr(tables: list[Table]) -> str:
123 """Render astropy tables to string as they are displayed in the CLI.
125 Output formatting matches ``printAstropyTables``.
127 Parameters
128 ----------
129 tables : `list` of `astropy.table.Table`
130 The tables to format.
132 Returns
133 -------
134 formatted : `str`
135 Tables formatted into a string.
136 """
137 ret = ""
138 for table in tables:
139 ret += "\n"
140 table.pformat()
141 ret += "\n"
142 return ret
145def printAstropyTables(tables: list[Table]) -> None:
146 """Print astropy tables to be displayed in the CLI.
148 Output formatting matches ``astropyTablesToStr``.
150 Parameters
151 ----------
152 tables : `list` of `astropy.table.Table`
153 The tables to print.
154 """
155 for table in tables:
156 print("")
157 table.pprint_all()
158 print("")
161def textTypeStr(multiple: bool) -> str:
162 """Get the text type string for CLI help documentation.
164 Parameters
165 ----------
166 multiple : `bool`
167 True if multiple text values are allowed, False if only one value is
168 allowed.
170 Returns
171 -------
172 textTypeStr : `str`
173 The type string to use.
174 """
175 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
178class ClickExitFailedNicely:
179 """Exit a Click command that failed.
181 This class is used to control the behavior when a click command has failed.
182 By default the exception will be logged and a non-zero exit status will
183 be used.
185 Parameters
186 ----------
187 exc_type : `type`
188 The type of the exception.
189 exc_value : `Exception`
190 The `Exception` object.
191 exc_tb : `types.TracebackType`
192 The traceback for this exception.
193 """
195 use_bad_status: bool = True
196 """Control how a bad command is exited. `True` indicates bad status and
197 `False` indicates a `click.ClickException`."""
199 def __init__(self, exc_type: type[BaseException], exc_value: BaseException, exc_tb: types.TracebackType):
200 self.exc_type = exc_type
201 self.exc_value = exc_value
202 self.exc_tb = self._clean_tb(exc_tb)
204 def _clean_tb(self, exc_tb: types.TracebackType) -> types.TracebackType:
205 if exc_tb.tb_next:
206 # Do not show the decorator in traceback.
207 exc_tb = exc_tb.tb_next
208 return exc_tb
210 def exit_click(self) -> None:
211 if self.use_bad_status:
212 self.exit_click_command_bad_status()
213 else:
214 self.exit_click_command_click_exception()
216 def exit_click_command_bad_status(self) -> None:
217 """Exit a click command with bad exit status and report log message."""
218 log.exception(
219 "Caught an exception, details are in traceback:",
220 exc_info=(self.exc_type, self.exc_value, self.exc_tb),
221 )
222 # Tell click to stop, this never returns.
223 click.get_current_context().exit(1)
225 def exit_click_command_click_exception(self) -> None:
226 """Exit a click command raising ClickException."""
227 tb = traceback.format_tb(self.exc_tb)
228 errmsg = "".join(tb) + str(self.exc_value)
229 raise click.ClickException(errmsg)
232class LogCliRunner(click.testing.CliRunner):
233 """A test runner to use when the logging system will be initialized by code
234 under test, calls CliLog.resetLog(), which undoes any logging setup that
235 was done with the CliLog interface.
237 lsst.log modules can not be set back to an uninitialized state (python
238 logging modules can be set back to NOTSET), instead they are set to
239 `CliLog.defaultLsstLogLevel`.
240 """
242 def invoke(self, *args: Any, **kwargs: Any) -> click.testing.Result:
243 # We want exceptions to be reported to the test runner rather than
244 # being converted to a simple exit status. The default is to
245 # use a logger but the click test infrastructure doesn't capture that
246 # in result.
247 with patch.object(ClickExitFailedNicely, "use_bad_status", False):
248 result = super().invoke(*args, **kwargs)
249 CliLog.resetLog()
250 if result.exception:
251 print("Failing command was: ", args)
252 return result
255def clickResultMsg(result: click.testing.Result) -> str:
256 """Get a standard assert message from a click result.
258 Parameters
259 ----------
260 result : click.testing.Result
261 The result object returned from `click.testing.CliRunner.invoke`.
263 Returns
264 -------
265 msg : `str`
266 The message string.
267 """
268 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
269 if result.exception:
270 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
271 return msg
274@contextmanager
275def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
276 """Context manager that creates (and then cleans up) an environment that
277 provides a CLI plugin command with the given name.
279 Parameters
280 ----------
281 runner : click.testing.CliRunner
282 The test runner to use to create the isolated filesystem.
283 commandModule : `str`
284 The importable module that the command can be imported from.
285 commandName : `str`
286 The name of the command being published to import.
287 """
288 with runner.isolated_filesystem():
289 with open("resources.yaml", "w") as f:
290 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
291 # Add a colon to the end of the path on the next line, this tests the
292 # case where the lookup in LoaderCLI._getPluginList generates an empty
293 # string in one of the list entries and verifies that the empty string
294 # is properly stripped out.
295 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
296 yield
299def addArgumentHelp(doc: str | None, helpText: str) -> str:
300 """Add a Click argument's help message to a function's documentation.
302 This is needed because click presents arguments in the order the argument
303 decorators are applied to a function, top down. But, the evaluation of the
304 decorators happens bottom up, so if arguments just append their help to the
305 function's docstring, the argument descriptions appear in reverse order
306 from the order they are applied in.
308 Parameters
309 ----------
310 doc : `str`
311 The function's docstring.
312 helpText : `str`
313 The argument's help string to be inserted into the function's
314 docstring.
316 Returns
317 -------
318 doc : `str`
319 Updated function documentation.
320 """
321 if doc is None: 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 doc = helpText
323 else:
324 # See click documentation for details:
325 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
326 # In short, text for the click command help can be truncated by putting
327 # "\f" in the docstring, everything after it should be removed
328 if "\f" in doc: 328 ↛ 329line 328 didn't jump to line 329 because the condition on line 328 was never true
329 doc = doc.split("\f")[0]
331 doclines = doc.splitlines()
332 # The function's docstring may span multiple lines, so combine the
333 # docstring from all the first lines until a blank line is encountered.
334 # (Lines after the first blank line will be argument help.)
335 while len(doclines) > 1 and doclines[1]:
336 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
337 # Add standard indent to help text for proper alignment with command
338 # function documentation:
339 helpText = " " + helpText
340 doclines.insert(1, helpText)
341 doclines.insert(1, "\n")
342 doc = "\n".join(doclines)
343 return doc
346def split_commas(
347 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None
348) -> tuple[str, ...]:
349 """Process a tuple of values, where each value may contain comma-separated
350 values, and return a single list of all the passed-in values.
352 This function can be passed to the 'callback' argument of a click.option to
353 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
354 the comma is inside ``[]`` there will be no splitting.
356 Parameters
357 ----------
358 context : `click.Context` or `None`
359 The current execution context. Unused, but Click always passes it to
360 callbacks.
361 param : `click.core.Option` or `None`
362 The parameter being handled. Unused, but Click always passes it to
363 callbacks.
364 values : `~collections.abc.Iterable` of `str` or `str`
365 All the values passed for this option. Strings may contain commas,
366 which will be treated as delimiters for separate values unless they
367 are within ``[]``.
369 Returns
370 -------
371 results : `tuple` [`str`]
372 The passed in values separated by commas where appropriate and
373 combined into a single tuple.
374 """
375 if values is None:
376 return ()
377 valueList = []
378 for value in ensure_iterable(values):
379 # If we have [, or ,] we do the slow split. If square brackets
380 # are not matching then that is likely a typo that should result
381 # in a warning.
382 opens = "["
383 closes = "]"
384 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
385 in_parens = False
386 current = ""
387 for c in value:
388 if c == opens:
389 if in_parens:
390 warnings.warn(
391 f"Found second opening {opens} without corresponding closing {closes}"
392 f" in {value!r}",
393 stacklevel=2,
394 )
395 in_parens = True
396 elif c == closes:
397 if not in_parens:
398 warnings.warn(
399 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
400 stacklevel=2,
401 )
402 in_parens = False
403 elif c == "," and not in_parens:
404 # Split on this comma.
405 valueList.append(current)
406 current = ""
407 continue
408 current += c
409 if in_parens:
410 warnings.warn(
411 f"Found opening {opens} that was never closed in {value!r}",
412 stacklevel=2,
413 )
414 if current:
415 valueList.append(current)
416 else:
417 # Use efficient split since no parens.
418 valueList.extend(value.split(","))
419 return tuple(valueList)
422def split_kv(
423 context: click.Context,
424 param: click.core.Option,
425 values: list[str],
426 *,
427 choice: click.Choice | None = None,
428 multiple: bool = True,
429 normalize: bool = False,
430 separator: str = "=",
431 unseparated_okay: bool = False,
432 return_type: type[dict] | type[tuple] = dict,
433 default_key: str | None = "",
434 reverse_kv: bool = False,
435 add_to_default: bool = False,
436) -> dict[str | None, str] | tuple[tuple[str | None, str], ...]:
437 """Process a tuple of values that are key-value pairs separated by a given
438 separator. Multiple pairs may be comma separated. Return a dictionary of
439 all the passed-in values.
441 This function can be passed to the 'callback' argument of a click.option to
442 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
444 Parameters
445 ----------
446 context : `click.Context` or `None`
447 The current execution context. Unused, but Click always passes it to
448 callbacks.
449 param : `click.core.Option` or `None`
450 The parameter being handled. Unused, but Click always passes it to
451 callbacks.
452 values : [`str`]
453 All the values passed for this option. Strings may contain commas,
454 which will be treated as delimiters for separate values.
455 choice : `click.Choice`, optional
456 If provided, verify each value is a valid choice using the provided
457 `click.Choice` instance. If None, no verification will be done. By
458 default `None`.
459 multiple : `bool`, optional
460 If true, the value may contain multiple comma-separated values. By
461 default True.
462 normalize : `bool`, optional
463 If True and ``choice.case_sensitive == False``, normalize the string
464 the user provided to match the choice's case. By default `False`.
465 separator : str, optional
466 The character that separates key-value pairs. May not be a comma or an
467 empty space (for space separators use Click's default implementation
468 for tuples; ``type=(str, str)``). By default "=".
469 unseparated_okay : `bool`, optional
470 If True, allow values that do not have a separator. They will be
471 returned in the values dict as a tuple of values in the key '', that
472 is: ``values[''] = (unseparated_values, )``. By default False.
473 return_type : `type`, must be `dict` or `tuple`
474 The type of the value that should be returned.
475 If `dict` then the returned object will be a dict, for each item in
476 values, the value to the left of the separator will be the key and the
477 value to the right of the separator will be the value.
478 If `tuple` then the returned object will be a tuple. Each item in the
479 tuple will be 2-item tuple, the first item will be the value to the
480 left of the separator and the second item will be the value to the
481 right. By default `dict`.
482 default_key : `str` or `None`
483 The key to use if a value is passed that is not a key-value pair.
484 `None` can imply no separator depending on how the results are handled.
485 (Passing values that are not key-value pairs requires
486 ``unseparated_okay`` to be `True`).
487 reverse_kv : bool
488 If true then for each item in values, the value to the left of the
489 separator is treated as the value and the value to the right of the
490 separator is treated as the key. By default `False`.
491 add_to_default : `bool`, optional
492 If True, then passed-in values will not overwrite the default value
493 unless the ``return_type`` is `dict` and passed-in value(s) have the
494 same key(s) as the default value.
496 Returns
497 -------
498 values : `dict` [`str`, `str`] or `tuple` [`tuple` [`str`, `str`], ...]
499 The passed-in values in dict form or tuple form.
501 Raises
502 ------
503 `click.ClickException`
504 Raised if the separator is not found in an entry, or if duplicate keys
505 are encountered.
506 """
508 def norm(val: str) -> str:
509 """If `normalize` is `True` and `choice` is not `None`, find the value
510 in the available choices and return the value as spelled in the
511 choices.
513 Assumes that val exists in choices; `split_kv` uses the `choice`
514 instance to verify val is a valid choice.
516 Parameters
517 ----------
518 val : `str`
519 Value to be found.
521 Returns
522 -------
523 val : `str`
524 The value that was found or the value that was given.
525 """
526 if normalize and choice is not None:
527 v = val.casefold()
528 for opt in choice.choices:
529 if opt.casefold() == v:
530 return opt
531 return val
533 class RetDict:
534 def __init__(self) -> None:
535 self.ret: dict[str | None, str] = {}
537 def add(self, key: str | None, val: str) -> None:
538 if reverse_kv:
539 key, val = val, key
540 self.ret[key] = val
542 def get(self) -> dict[str | None, str]:
543 return self.ret
545 class RetTuple:
546 def __init__(self) -> None:
547 self.ret: list[tuple[str | None, str]] = []
549 def add(self, key: str | None, val: str) -> None:
550 if reverse_kv:
551 key, val = val, key
552 self.ret.append((key, val))
554 def get(self) -> tuple[tuple[str | None, str], ...]:
555 return tuple(self.ret)
557 if separator in (",", " "):
558 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
559 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
561 if add_to_default:
562 default = param.get_default(context)
563 if default and default != _CLICK_UNSET_SENTINEL:
564 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy
566 ret: RetDict | RetTuple
567 if return_type is dict:
568 ret = RetDict()
569 elif return_type is tuple:
570 ret = RetTuple()
571 else:
572 raise click.ClickException(
573 message=f"Internal error: invalid return type '{return_type}' for split_kv."
574 )
575 if multiple:
576 vals = split_commas(context, param, vals)
577 for val in ensure_iterable(vals):
578 if unseparated_okay and separator not in val:
579 if choice is not None:
580 choice(val) # will raise if val is an invalid choice
581 ret.add(default_key, norm(val))
582 else:
583 try:
584 k, v = val.split(separator)
585 if choice is not None:
586 choice(v) # will raise if val is an invalid choice
587 except ValueError as e:
588 raise click.ClickException(
589 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
590 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
591 ) from None
592 ret.add(k, norm(v))
593 return ret.get()
596def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
597 """Convert a value to upper case.
599 Parameters
600 ----------
601 context : `click.Context`
602 Context given by Click.
603 param : `click.core.Option`
604 Provided by Click. Ignored.
605 value : `str`
606 The value to be converted.
608 Returns
609 -------
610 str
611 A copy of the passed-in value, converted to upper case.
612 """
613 return value.upper()
616def unwrap(val: str) -> str:
617 """Remove newlines and leading whitespace from a multi-line string with
618 a consistent indentation level.
620 The first line of the string may be only a newline or may contain text
621 followed by a newline, either is ok. After the first line, each line must
622 begin with a consistant amount of whitespace. So, content of a
623 triple-quoted string may begin immediately after the quotes, or the string
624 may start with a newline. Each line after that must be the same amount of
625 indentation/whitespace followed by text and a newline. The last line may
626 end with a new line but is not required to do so.
628 Parameters
629 ----------
630 val : `str`
631 The string to change.
633 Returns
634 -------
635 strippedString : `str`
636 The string with newlines, indentation, and leading and trailing
637 whitespace removed.
638 """
640 def splitSection(val: str) -> str:
641 if not val.startswith("\n"): 641 ↛ 645line 641 didn't jump to line 645 because the condition on line 641 was always true
642 firstLine, _, val = val.partition("\n")
643 firstLine += " "
644 else:
645 firstLine = ""
646 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
648 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
651class option_section: # noqa: N801
652 """Decorator to add a section label between options in the help text of a
653 command.
655 Parameters
656 ----------
657 sectionText : `str`
658 The text to print in the section identifier.
659 """
661 def __init__(self, sectionText: str) -> None:
662 self.sectionText = "\n" + sectionText
664 def __call__(self, f: Any) -> click.Option:
665 # Generate a parameter declaration that will be unique for this
666 # section.
667 return click.option(
668 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
669 )(f)
672class MWPath(click.Path):
673 """Overrides `click.Path` to implement file-does-not-exist checking.
675 Changes the definition of ``exists`` so that `True` indicates the location
676 (file or directory) must exist, `False` indicates the location must *not*
677 exist, and `None` indicates that the file may exist or not. The standard
678 definition for the `click.Path` ``exists`` parameter is that for `True` a
679 location must exist, but `False` means it is not required to exist (not
680 that it is required to not exist).
682 Parameters
683 ----------
684 exists : `bool` or `None`, optional
685 If `True`, the location (file or directory) indicated by the caller
686 must exist. If `False` the location must not exist. If `None`, the
687 location may exist or not.
688 file_okay : `bool`, optional
689 Allow a file as a value.
690 dir_okay : `bool`, optional
691 Allow a directory as a value.
692 writable : `bool`, optional
693 If `True`, a writable check is performed.
694 readable : `bool`, optional
695 If `True`, a readable check is performed.
696 resolve_path : `bool`, optional
697 Resolve the path.
698 allow_dash : `bool`, optional
699 Allow single dash as value to mean a standard stream.
700 path_type : `type` or `None`, optional
701 Convert the incoming value to this type.
703 Notes
704 -----
705 All parameters other than ``exists`` come directly from `click.Path`.
706 """
708 def __init__(
709 self,
710 exists: bool | None = None,
711 file_okay: bool = True,
712 dir_okay: bool = True,
713 writable: bool = False,
714 readable: bool = True,
715 resolve_path: bool = False,
716 allow_dash: bool = False,
717 path_type: type | None = None,
718 ):
719 self.mustNotExist = exists is False
720 if exists is None: 720 ↛ 722line 720 didn't jump to line 722 because the condition on line 720 was always true
721 exists = False
722 super().__init__(
723 exists=exists,
724 file_okay=file_okay,
725 dir_okay=dir_okay,
726 writable=writable,
727 readable=readable,
728 resolve_path=resolve_path,
729 allow_dash=allow_dash,
730 path_type=path_type,
731 )
733 def convert(
734 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None
735 ) -> Any:
736 """Convert values through types.
738 Called by `click.ParamType` to "convert values through types".
739 `click.Path` uses this step to verify Path conditions.
741 Parameters
742 ----------
743 value : `str` or `os.PathLike`
744 File path.
745 param : `click.Parameter`
746 Parameters provided by Click.
747 ctx : `click.Context`
748 Context provided by Click.
749 """
750 if self.mustNotExist and os.path.exists(value):
751 self.fail(f'Path "{value}" should not exist.')
752 return super().convert(value, param, ctx)
755class MWOption(click.Option):
756 """Overrides click.Option with desired behaviors."""
758 def make_metavar(self, ctx: click.Context | None = None) -> str:
759 """Make the metavar for the help menu.
761 Parameters
762 ----------
763 ctx : `click.Context` or `None`
764 Context from the command.
766 Notes
767 -----
768 Overrides `click.Option.make_metavar`.
769 Adds a space and an ellipsis after the metavar name if
770 the option accepts multiple inputs, otherwise defers to the base
771 implementation.
773 By default click does not add an ellipsis when multiple is True and
774 nargs is 1. And when nargs does not equal 1 click adds an ellipsis
775 without a space between the metavar and the ellipsis, but we prefer a
776 space between.
778 Does not get called for some option types (e.g. flag) so metavar
779 transformation that must apply to all types should be applied in
780 get_help_record.
781 """
782 if _click_make_metavar_has_context:
783 metavar = super().make_metavar(ctx=ctx) # type: ignore
784 else:
785 metavar = super().make_metavar() # type: ignore
786 if self.multiple and self.nargs == 1:
787 metavar += " ..."
788 elif self.nargs != 1:
789 metavar = f"{metavar[:-3]} ..."
790 return metavar
793class MWArgument(click.Argument):
794 """Overrides click.Argument with desired behaviors."""
796 def make_metavar(self, ctx: click.Context | None = None) -> str:
797 """Make the metavar for the help menu.
799 Parameters
800 ----------
801 ctx : `click.Context` or `None`
802 Context from the command.
804 Notes
805 -----
806 Overrides `click.Option.make_metavar`.
807 Always adds a space and an ellipsis (' ...') after the
808 metavar name if the option accepts multiple inputs.
810 By default click adds an ellipsis without a space between the metavar
811 and the ellipsis, but we prefer a space between.
813 Returns
814 -------
815 metavar : `str`
816 The metavar value.
817 """
818 if _click_make_metavar_has_context:
819 metavar = super().make_metavar(ctx=ctx) # type: ignore
820 else:
821 metavar = super().make_metavar() # type: ignore
822 if self.nargs != 1:
823 metavar = f"{metavar[:-3]} ..."
824 return metavar
827class OptionSection(MWOption):
828 """Implements an Option that prints a section label in the help text and
829 does not pass any value to the command function.
831 This class does a bit of hackery to add a section label to a click command
832 help output: first, ``expose_value`` is set to `False` so that no value is
833 passed to the command function. Second, this class overrides
834 `click.Option.get_help_record` to return the section label string without
835 any prefix so that it stands out as a section label.
837 This class overrides the hidden attribute because our documentation build
838 tool, sphinx-click, implements its own `get_help_record` function which
839 builds the record from other option values (e.g. ``name``, ``opts``), which
840 breaks the hack we use to make `get_help_record` only return the
841 ``sectionText``. Fortunately, Click gets the value of `hidden` inside the
842 `click.Option`'s `get_help_record`, and sphinx-click calls ``opt.hidden``
843 before entering its ``_get_help_record`` function. So, making the hidden
844 property return True hides this option from sphinx-click, while allowing
845 the section text to be returned by our `get_help_record` method when using
846 Click.
848 The intention for this implementation is to do minimally invasive overrides
849 of the click classes so as to be robust and easy to fix if the click
850 internals change.
852 Parameters
853 ----------
854 sectionName : `str`
855 The parameter declaration for this option. It is not shown to the user,
856 it must be unique within the command. If using the ``section``
857 decorator to add a section to a command's options, the section name is
858 auto-generated.
859 sectionText : `str`
860 The text to print in the section identifier.
861 """
863 @property
864 def hidden(self) -> bool:
865 return True
867 @hidden.setter
868 def hidden(self, val: Any) -> None:
869 pass
871 def __init__(self, sectionName: str, sectionText: str) -> None:
872 super().__init__(sectionName, expose_value=False)
873 self.sectionText = sectionText
875 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
876 return (self.sectionText, "")
879class MWOptionDecorator:
880 """Wraps the click.option decorator to enable shared options to be declared
881 and allows inspection of the shared option.
883 Parameters
884 ----------
885 *param_decls : `typing.Any`
886 Parameters to be stored in the option.
887 **kwargs : `typing.Any`
888 Keyword arguments for the option.
889 """
891 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
892 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) # type: ignore
893 opt = click.Option(param_decls, **kwargs)
894 self._name = opt.name
895 self._opts = opt.opts
897 def name(self) -> str:
898 """Get the name that will be passed to the command function for this
899 option.
900 """
901 return cast(str, self._name)
903 def opts(self) -> list[str]:
904 """Get the flags that will be used for this option on the command
905 line.
906 """
907 return self._opts
909 @property
910 def help(self) -> str:
911 """Get the help text for this option. Returns an empty string if no
912 help was defined.
913 """
914 return self.partialOpt.keywords.get("help", "")
916 def __call__(self, *args: Any, **kwargs: Any) -> Any:
917 return self.partialOpt(*args, **kwargs)
920class MWArgumentDecorator:
921 """Wraps the click.argument decorator to enable shared arguments to be
922 declared.
924 Parameters
925 ----------
926 *param_decls : `typing.Any`
927 Parameters to be stored in the argument.
928 **kwargs : `typing.Any`
929 Keyword arguments for the argument.
930 """
932 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
933 self._helpText = kwargs.pop("help", None)
934 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
936 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
937 def decorator(f: Any) -> Any:
938 if help is not None:
939 self._helpText = help
940 if self._helpText:
941 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
942 return self.partialArg(*args, **kwargs)(f)
944 return decorator
947class MWCommand(click.Command):
948 """Command subclass that stores a copy of the args list for use by the
949 command.
951 Parameters
952 ----------
953 *args : `typing.Any`
954 Arguments for `click.Command`.
955 **kwargs : `typing.Any`
956 Keyword arguments for `click.Command`.
957 """
959 name = "butler"
960 extra_epilog: str | None = None
962 def __init__(self, *args: Any, **kwargs: Any) -> None:
963 # wrap callback method with catch_and_exit decorator
964 callback = kwargs.get("callback")
965 if callback is not None: 965 ↛ 968line 965 didn't jump to line 968 because the condition on line 965 was always true
966 kwargs = kwargs.copy()
967 kwargs["callback"] = catch_and_exit(callback)
968 super().__init__(*args, **kwargs)
970 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
971 """Capture the command line options and arguments.
973 See details about what is captured and the order in which it is stored
974 in the documentation of `MWCtxObj`.
976 Parameters
977 ----------
978 ctx : `click.Context`
979 The current Context.
980 args : `list` [`str`]
981 The list of arguments from the command line, split at spaces but
982 not at separators (like "=").
983 """
984 parser = self.make_parser(ctx)
985 opts, _, param_order = parser.parse_args(args=list(args))
986 # `param_order` is a list of click.Option and click.Argument, there is
987 # one item for each time the Option or Argument was used on the
988 # command line. Options will precede Arguments, within each sublist
989 # they are in the order they were used on the command line. Note that
990 # click.Option and click.Argument do not contain the value from the
991 # command line; values are in `opts`.
992 #
993 # `opts` is a dict where the key is the argument name to the
994 # click.Command function, this name matches the `click.Option.name` or
995 # `click.Argument.name`. For Options, an item will only be present if
996 # the Option was used on the command line. For Arguments, an item will
997 # always be present and if no value was provided on the command line
998 # the value will be `None`. If the option accepts multiple values, the
999 # value in `opts` is a tuple, otherwise it is a single item.
1000 next_idx: Counter = Counter()
1001 captured_args = []
1002 for param in param_order:
1003 if isinstance(param, click.Option):
1004 param_name = cast(str, param.name)
1005 if param.multiple:
1006 val = opts[param_name][next_idx[param_name]]
1007 next_idx[param_name] += 1
1008 else:
1009 val = opts[param_name]
1010 if param.is_flag:
1011 # Bool options store their True flags in opts and their
1012 # False flags in secondary_opts.
1013 if val:
1014 flag = max(param.opts, key=len)
1015 else:
1016 flag = max(param.secondary_opts, key=len)
1017 captured_args.append(flag)
1018 else:
1019 captured_args.append(max(param.opts, key=len))
1020 captured_args.append(val)
1021 elif isinstance(param, click.Argument):
1022 param_name = cast(str, param.name)
1023 opt = opts[param_name]
1024 if opt is not None and opt != _CLICK_UNSET_SENTINEL:
1025 captured_args.append(opt)
1026 else:
1027 raise AssertionError("All parameters should be an Option or an Argument")
1028 MWCtxObj.getFrom(ctx).args = captured_args
1030 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
1031 """Given a context and a list of arguments this creates the parser and
1032 parses the arguments, then modifies the context as necessary. This is
1033 automatically invoked by make_context().
1035 This function overrides `click.Command.parse_args`.
1037 The call to `_capture_args` in this override stores the arguments
1038 (option names, option value, and argument values) that were used by the
1039 caller on the command line in the context object. These stored
1040 arguments can be used by the command function, e.g. to process options
1041 in the order they appeared on the command line (``pipetask`` uses this
1042 feature to create pipeline actions in an order from different options).
1044 Parameters
1045 ----------
1046 ctx : `click.core.Context`
1047 The current Context.
1048 args : `list` [`str`]
1049 The list of arguments from the command line, split at spaces but
1050 not at separators (like "=").
1051 """
1052 self._capture_args(ctx, args)
1053 return super().parse_args(ctx, args)
1055 @property
1056 def epilog(self) -> str | None:
1057 """Override the epilog attribute to add extra_epilog (if defined by a
1058 subclass) to the end of any epilog provided by a subcommand.
1059 """
1060 ret = self._epilog if self._epilog else ""
1061 if self.extra_epilog:
1062 if ret:
1063 ret += "\n\n"
1064 ret += self.extra_epilog
1065 return ret
1067 @epilog.setter
1068 def epilog(self, val: str | None) -> None:
1069 self._epilog = val
1072class ButlerCommand(MWCommand):
1073 """Command subclass with butler-command specific overrides."""
1075 extra_epilog = "See 'butler --help' for more options."
1078class OptionGroup:
1079 """Base class for an option group decorator. Requires the option group
1080 subclass to have a property called ``decorator``.
1081 """
1083 decorators: list[Any]
1085 def __call__(self, f: Any) -> Any:
1086 for decorator in reversed(self.decorators):
1087 f = decorator(f)
1088 return f
1091class MWCtxObj:
1092 """Helper object for managing the `click.Context.obj` parameter, allows
1093 obj data to be managed in a consistent way.
1095 `click.Context.obj` defaults to None. ``MWCtxObj.getFrom(ctx)`` can be used
1096 to initialize the obj if needed and return a new or existing `MWCtxObj`.
1098 The `args` attribute contains a list of options, option values, and
1099 argument values that is similar to the list of arguments and options that
1100 were passed in on the command line, with differences noted below:
1102 * Option names and option values are first in the list, and argument
1103 values come last. The order of options and option values is preserved
1104 within the options. The order of argument values is preserved.
1106 * The longest option name is used for the option in the `args` list, e.g.
1107 if an option accepts both short and long names ``"-o / --option"`` and
1108 the short option name ``"-o"`` was used on the command line, the longer
1109 name will be the one that appears in ``args``.
1111 * A long option name (which begins with two dashes ``"--"``) and its value
1112 may be separated by an equal sign; the name and value are split at the
1113 equal sign and it is removed. In ``args``, the option is in one list
1114 item, and the option value (without the equal sign) is in the next list
1115 item. e.g. ``"--option=foo"`` and ``"--option foo"`` both become
1116 ``["--opt", "foo"]`` in ``args``.
1118 * A short option name, (which begins with one dash ``"-"``) and its value
1119 are split immediately after the short option name, and if there is
1120 whitespace between the short option name and its value it is removed.
1121 Everything after the short option name (excluding whitespace) is included
1122 in the value. If the ``Option`` has a long name, the long name will be
1123 used in ``args`` e.g. for the option ``"-o / --option"``: ``"-ofoo"`` and
1124 ``"-o foo"`` become ``["--option", "foo"]``, and (note!) ``"-o=foo"``
1125 will become ``["--option", "=foo"]`` (because everything after the short
1126 option name, except whitespace, is used for the value (as is standard
1127 with unix command line tools).
1129 Attributes
1130 ----------
1131 args : `list` [`str`]
1132 A list of options, option values, and arguments similar to those that
1133 were passed in on the command line. See comments about captured options
1134 & arguments above.
1135 """
1137 def __init__(self) -> None:
1138 self.args = None
1140 @staticmethod
1141 def getFrom(ctx: click.Context) -> Any:
1142 """If needed, initialize ``ctx.obj`` with a new `MWCtxObj`, and return
1143 the new or already existing `MWCtxObj`.
1145 Parameters
1146 ----------
1147 ctx : `click.Context`
1148 Context provided by Click.
1149 """
1150 if ctx.obj is not None:
1151 return ctx.obj
1152 ctx.obj = MWCtxObj()
1153 return ctx.obj
1156def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
1157 """Read additional values from the supplied YAML file.
1159 Parameters
1160 ----------
1161 ctx : `click.Context`
1162 The context for the click operation. Used to extract the subcommand
1163 name and translate option & argument names.
1164 param : `str`
1165 The parameter name.
1166 value : `object`
1167 The value of the parameter.
1168 """
1170 def _name_for_option(ctx: click.Context, option: str) -> str:
1171 """Use a CLI option name to find the name of the argument to the
1172 command function.
1174 Parameters
1175 ----------
1176 ctx : `click.Context`
1177 The context for the click operation.
1178 option : `str`
1179 The option/argument name from the yaml file.
1181 Returns
1182 -------
1183 name : str
1184 The name of the argument to use when calling the click.command
1185 function, as it should appear in the `ctx.default_map`.
1187 Raises
1188 ------
1189 RuntimeError
1190 Raised if the option name from the yaml file does not exist in the
1191 command parameters. This catches misspellings and incorrect usage
1192 in the yaml file.
1193 """
1194 for param in ctx.command.params:
1195 # Remove leading dashes: they are not used for option names in the
1196 # yaml file.
1197 if option in [opt.lstrip("-") for opt in param.opts]:
1198 return cast(str, param.name)
1199 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
1201 ctx.default_map = ctx.default_map or {}
1202 cmd_name = ctx.info_name
1203 assert cmd_name is not None, "command name cannot be None"
1204 if value:
1205 try:
1206 overrides = _read_yaml_presets(value, cmd_name)
1207 options = list(overrides.keys())
1208 for option in options:
1209 name = _name_for_option(ctx, option)
1210 if name == option:
1211 continue
1212 overrides[name] = overrides.pop(option)
1213 except Exception as e:
1214 raise click.BadOptionUsage(
1215 option_name=param,
1216 message=f"Error reading overrides file: {e}",
1217 ctx=ctx,
1218 ) from None
1219 # Override the defaults for this subcommand
1220 ctx.default_map.update(overrides)
1221 return
1224def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]:
1225 """Read file command line overrides from YAML config file.
1227 Parameters
1228 ----------
1229 file_uri : `str`
1230 URI of override YAML file containing the command line overrides.
1231 They should be grouped by command name.
1232 cmd_name : `str`
1233 The subcommand name that is being modified.
1235 Returns
1236 -------
1237 overrides : `dict` of [`str`, Any]
1238 The relevant command line options read from the override file.
1239 """
1240 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1241 config = Config(file_uri)
1242 return config[cmd_name]
1245def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1246 """Sort an astropy table.
1248 Prioritization is given to columns in this order:
1250 1. the provided named columns
1251 2. spatial and temporal columns
1252 3. the rest of the columns.
1254 The table is sorted in-place, and is also returned for convenience.
1256 Parameters
1257 ----------
1258 table : `astropy.table.Table`
1259 The table to sort.
1260 dimensions : `list` [``Dimension``]
1261 The dimensions of the dataIds in the table (the dimensions should be
1262 the same for all the dataIds). Used to determine if the column is
1263 spatial, temporal, or neither.
1264 sort_first : `list` [`str`]
1265 The names of columns that should be sorted first, before spatial and
1266 temporal columns.
1268 Returns
1269 -------
1270 `astropy.table.Table`
1271 For convenience, the table that has been sorted.
1272 """
1273 # For sorting we want to ignore the id
1274 # We also want to move temporal or spatial dimensions earlier
1275 sort_first = sort_first or []
1276 sort_early: list[str] = []
1277 sort_late: list[str] = []
1278 for dim in dimensions:
1279 if dim.spatial or dim.temporal:
1280 sort_early.extend(dim.required.names)
1281 else:
1282 sort_late.append(str(dim))
1283 sort_keys = sort_first + sort_early + sort_late
1284 # The required names above means that we have the possibility of
1285 # repeats of sort keys. Now have to remove them
1286 # (order is retained by dict creation).
1287 sort_keys = list(dict.fromkeys(sort_keys).keys())
1289 table.sort(sort_keys)
1290 return table
1293def catch_and_exit(func: Callable) -> Callable:
1294 """Catch all exceptions, prints an exception traceback
1295 and signals click to exit.
1297 Use as decorator.
1299 Parameters
1300 ----------
1301 func : `collections.abc.Callable`
1302 The function to be decorated.
1304 Returns
1305 -------
1306 `collections.abc.Callable`
1307 The decorated function.
1308 """
1310 @wraps(func)
1311 def inner(*args: Any, **kwargs: Any) -> None:
1312 try:
1313 func(*args, **kwargs)
1314 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1315 # this is handled by click itself
1316 raise
1317 except Exception:
1318 exc_type, exc_value, exc_tb = sys.exc_info()
1319 assert exc_type is not None
1320 assert exc_value is not None
1321 assert exc_tb is not None
1322 exit_hdl = ClickExitFailedNicely(exc_type, exc_value, exc_tb)
1323 exit_hdl.exit_click()
1325 return inner