Coverage for python / lsst / daf / butler / cli / utils.py: 33%
389 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-05-01 08:18 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "ButlerCommand",
31 "LogCliRunner",
32 "MWArgument",
33 "MWArgumentDecorator",
34 "MWCommand",
35 "MWCtxObj",
36 "MWOption",
37 "MWOptionDecorator",
38 "MWPath",
39 "OptionGroup",
40 "OptionSection",
41 "addArgumentHelp",
42 "astropyTablesToStr",
43 "catch_and_exit",
44 "clickResultMsg",
45 "command_test_env",
46 "option_section",
47 "printAstropyTables",
48 "sortAstropyTable",
49 "split_commas",
50 "split_kv",
51 "textTypeStr",
52 "to_upper",
53 "unwrap",
54 "yaml_presets",
55)
58import importlib.metadata
59import itertools
60import logging
61import os
62import re
63import sys
64import textwrap
65import traceback
66import types
67import uuid
68import warnings
69from collections import Counter
70from collections.abc import Callable, Iterable, Iterator
71from contextlib import contextmanager
72from functools import partial, wraps
73from typing import TYPE_CHECKING, Any, cast
74from unittest.mock import patch
76import click
77import click.core
78import click.exceptions
79import click.testing
80import yaml
81from packaging.version import Version
83from lsst.utils.iteration import ensure_iterable
85from .._config import Config
86from .cliLog import CliLog
88if TYPE_CHECKING:
89 from astropy.table import Table
91 from lsst.daf.butler import Dimension
93_click_version = Version(importlib.metadata.version("click"))
94if _click_version >= Version("8.2.0"): 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true
95 _click_make_metavar_has_context = True
96else:
97 _click_make_metavar_has_context = False
99# Starting from Click 8.3.0, a special `UNSET` sentinel value is used to
100# indicate the absence of a default value for a parameter. Prior to 8.3.0,
101# they just used `None`.
102_CLICK_UNSET_SENTINEL = getattr(click.core, "UNSET", None)
104log = logging.getLogger(__name__)
106# This is used as the metavar argument to Options that accept multiple string
107# inputs, which may be comma-separarated. For example:
108# --my-opt foo,bar --my-opt baz.
109# Other arguments to the Option should include multiple=true and
110# callback=split_kv.
111typeStrAcceptsMultiple = "TEXT ..."
112typeStrAcceptsSingle = "TEXT"
114# The standard help string for the --where option when it takes a WHERE clause.
115where_help = (
116 "A string expression similar to a SQL WHERE clause. May involve any column of a "
117 "dimension table or a dimension name as a shortcut for the primary key column of a "
118 "dimension table."
119)
122def astropyTablesToStr(tables: list[Table]) -> str:
123 """Render astropy tables to string as they are displayed in the CLI.
125 Output formatting matches ``printAstropyTables``.
127 Parameters
128 ----------
129 tables : `list` of `astropy.table.Table`
130 The tables to format.
132 Returns
133 -------
134 formatted : `str`
135 Tables formatted into a string.
136 """
137 ret = ""
138 for table in tables:
139 ret += "\n"
140 table.pformat()
141 ret += "\n"
142 return ret
145def printAstropyTables(tables: list[Table]) -> None:
146 """Print astropy tables to be displayed in the CLI.
148 Output formatting matches ``astropyTablesToStr``.
150 Parameters
151 ----------
152 tables : `list` of `astropy.table.Table`
153 The tables to print.
154 """
155 for table in tables:
156 print("")
157 table.pprint_all()
158 print("")
161def textTypeStr(multiple: bool) -> str:
162 """Get the text type string for CLI help documentation.
164 Parameters
165 ----------
166 multiple : `bool`
167 True if multiple text values are allowed, False if only one value is
168 allowed.
170 Returns
171 -------
172 textTypeStr : `str`
173 The type string to use.
174 """
175 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
178class ClickExitFailedNicely:
179 """Exit a Click command that failed.
181 This class is used to control the behavior when a click command has failed.
182 By default the exception will be logged and a non-zero exit status will
183 be used.
185 Parameters
186 ----------
187 exc_type : `type`
188 The type of the exception.
189 exc_value : `Exception`
190 The `Exception` object.
191 exc_tb : `types.TracebackType`
192 The traceback for this exception.
193 """
195 use_bad_status: bool = True
196 """Control how a bad command is exited. `True` indicates bad status and
197 `False` indicates a `click.ClickException`."""
199 def __init__(self, exc_type: type[BaseException], exc_value: BaseException, exc_tb: types.TracebackType):
200 self.exc_type = exc_type
201 self.exc_value = exc_value
202 self.exc_tb = self._clean_tb(exc_tb)
204 def _clean_tb(self, exc_tb: types.TracebackType) -> types.TracebackType:
205 if exc_tb.tb_next:
206 # Do not show the decorator in traceback.
207 exc_tb = exc_tb.tb_next
208 return exc_tb
210 def exit_click(self) -> None:
211 if self.use_bad_status:
212 self.exit_click_command_bad_status()
213 else:
214 self.exit_click_command_click_exception()
216 def exit_click_command_bad_status(self) -> None:
217 """Exit a click command with bad exit status and report log message."""
218 log.exception(
219 "Caught an exception, details are in traceback:",
220 exc_info=(self.exc_type, self.exc_value, self.exc_tb),
221 )
222 # Tell click to stop, this never returns.
223 click.get_current_context().exit(1)
225 def exit_click_command_click_exception(self) -> None:
226 """Exit a click command raising ClickException."""
227 tb = traceback.format_tb(self.exc_tb)
228 errmsg = "".join(tb) + str(self.exc_value)
229 raise click.ClickException(errmsg)
232class LogCliRunner(click.testing.CliRunner):
233 """A test runner to use when the logging system will be initialized by code
234 under test, calls CliLog.resetLog(), which undoes any logging setup that
235 was done with the CliLog interface.
237 lsst.log modules can not be set back to an uninitialized state (python
238 logging modules can be set back to NOTSET), instead they are set to
239 `CliLog.defaultLsstLogLevel`.
240 """
242 def invoke(self, *args: Any, **kwargs: Any) -> click.testing.Result:
243 # We want exceptions to be reported to the test runner rather than
244 # being converted to a simple exit status. The default is to
245 # use a logger but the click test infrastructure doesn't capture that
246 # in result.
247 with patch.object(ClickExitFailedNicely, "use_bad_status", False):
248 result = super().invoke(*args, **kwargs)
249 CliLog.resetLog()
250 if result.exception:
251 print("Failing command was: ", args)
252 return result
255def clickResultMsg(result: click.testing.Result) -> str:
256 """Get a standard assert message from a click result.
258 Parameters
259 ----------
260 result : click.testing.Result
261 The result object returned from `click.testing.CliRunner.invoke`.
263 Returns
264 -------
265 msg : `str`
266 The message string.
267 """
268 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
269 if result.exception:
270 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
271 return msg
274@contextmanager
275def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
276 """Context manager that creates (and then cleans up) an environment that
277 provides a CLI plugin command with the given name.
279 Parameters
280 ----------
281 runner : click.testing.CliRunner
282 The test runner to use to create the isolated filesystem.
283 commandModule : `str`
284 The importable module that the command can be imported from.
285 commandName : `str`
286 The name of the command being published to import.
287 """
288 with runner.isolated_filesystem():
289 with open("resources.yaml", "w") as f:
290 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
291 # Add a colon to the end of the path on the next line, this tests the
292 # case where the lookup in LoaderCLI._getPluginList generates an empty
293 # string in one of the list entries and verifies that the empty string
294 # is properly stripped out.
295 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
296 yield
299def addArgumentHelp(doc: str | None, helpText: str) -> str:
300 """Add a Click argument's help message to a function's documentation.
302 This is needed because click presents arguments in the order the argument
303 decorators are applied to a function, top down. But, the evaluation of the
304 decorators happens bottom up, so if arguments just append their help to the
305 function's docstring, the argument descriptions appear in reverse order
306 from the order they are applied in.
308 Parameters
309 ----------
310 doc : `str`
311 The function's docstring.
312 helpText : `str`
313 The argument's help string to be inserted into the function's
314 docstring.
316 Returns
317 -------
318 doc : `str`
319 Updated function documentation.
320 """
321 if doc is None: 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 doc = helpText
323 else:
324 # See click documentation for details:
325 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
326 # In short, text for the click command help can be truncated by putting
327 # "\f" in the docstring, everything after it should be removed
328 if "\f" in doc: 328 ↛ 329line 328 didn't jump to line 329 because the condition on line 328 was never true
329 doc = doc.split("\f")[0]
331 doclines = doc.splitlines()
332 # The function's docstring may span multiple lines, so combine the
333 # docstring from all the first lines until a blank line is encountered.
334 # (Lines after the first blank line will be argument help.)
335 while len(doclines) > 1 and doclines[1]:
336 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
337 # Add standard indent to help text for proper alignment with command
338 # function documentation:
339 helpText = " " + helpText
340 doclines.insert(1, helpText)
341 doclines.insert(1, "\n")
342 doc = "\n".join(doclines)
343 return doc
346def split_commas(
347 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None
348) -> tuple[str, ...]:
349 """Process a tuple of values, where each value may contain comma-separated
350 values, and return a single list of all the passed-in values.
352 This function can be passed to the 'callback' argument of a click.option to
353 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
354 the comma is inside ``[]`` there will be no splitting.
356 Parameters
357 ----------
358 context : `click.Context` or `None`
359 The current execution context. Unused, but Click always passes it to
360 callbacks.
361 param : `click.core.Option` or `None`
362 The parameter being handled. Unused, but Click always passes it to
363 callbacks.
364 values : `~collections.abc.Iterable` of `str` or `str`
365 All the values passed for this option. Strings may contain commas,
366 which will be treated as delimiters for separate values unless they
367 are within ``[]``.
369 Returns
370 -------
371 results : `tuple` [`str`]
372 The passed in values separated by commas where appropriate and
373 combined into a single tuple.
374 """
375 if values is None:
376 return ()
377 valueList = []
378 for value in ensure_iterable(values):
379 # If we have [, or ,] we do the slow split. If square brackets
380 # are not matching then that is likely a typo that should result
381 # in a warning.
382 opens = "["
383 closes = "]"
384 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
385 in_parens = False
386 current = ""
387 for c in value:
388 if c == opens:
389 if in_parens:
390 warnings.warn(
391 f"Found second opening {opens} without corresponding closing {closes}"
392 f" in {value!r}",
393 stacklevel=2,
394 )
395 in_parens = True
396 elif c == closes:
397 if not in_parens:
398 warnings.warn(
399 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
400 stacklevel=2,
401 )
402 in_parens = False
403 elif c == "," and not in_parens:
404 # Split on this comma.
405 valueList.append(current)
406 current = ""
407 continue
408 current += c
409 if in_parens:
410 warnings.warn(
411 f"Found opening {opens} that was never closed in {value!r}",
412 stacklevel=2,
413 )
414 if current:
415 valueList.append(current)
416 else:
417 # Use efficient split since no parens.
418 valueList.extend(value.split(","))
419 return tuple(valueList)
422def split_kv(
423 context: click.Context,
424 param: click.core.Option,
425 values: list[str],
426 *,
427 choice: click.Choice | None = None,
428 multiple: bool = True,
429 normalize: bool = False,
430 separator: str = "=",
431 unseparated_okay: bool = False,
432 return_type: type[dict] | type[tuple] = dict,
433 default_key: str | None = "",
434 reverse_kv: bool = False,
435 add_to_default: bool = False,
436) -> dict[str | None, str] | tuple[tuple[str | None, str], ...]:
437 """Process a tuple of values that are key-value pairs separated by a given
438 separator. Multiple pairs may be comma separated. Return a dictionary of
439 all the passed-in values.
441 This function can be passed to the 'callback' argument of a click.option to
442 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
444 Parameters
445 ----------
446 context : `click.Context` or `None`
447 The current execution context. Unused, but Click always passes it to
448 callbacks.
449 param : `click.core.Option` or `None`
450 The parameter being handled. Unused, but Click always passes it to
451 callbacks.
452 values : [`str`]
453 All the values passed for this option. Strings may contain commas,
454 which will be treated as delimiters for separate values.
455 choice : `click.Choice`, optional
456 If provided, verify each value is a valid choice using the provided
457 `click.Choice` instance. If None, no verification will be done. By
458 default `None`.
459 multiple : `bool`, optional
460 If true, the value may contain multiple comma-separated values. By
461 default True.
462 normalize : `bool`, optional
463 If True and ``choice.case_sensitive == False``, normalize the string
464 the user provided to match the choice's case. By default `False`.
465 separator : str, optional
466 The character that separates key-value pairs. May not be a comma or an
467 empty space (for space separators use Click's default implementation
468 for tuples; ``type=(str, str)``). By default "=".
469 unseparated_okay : `bool`, optional
470 If True, allow values that do not have a separator. They will be
471 returned in the values dict as a tuple of values in the key '', that
472 is: ``values[''] = (unseparated_values, )``. By default False.
473 return_type : `type`, must be `dict` or `tuple`
474 The type of the value that should be returned.
475 If `dict` then the returned object will be a dict, for each item in
476 values, the value to the left of the separator will be the key and the
477 value to the right of the separator will be the value.
478 If `tuple` then the returned object will be a tuple. Each item in the
479 tuple will be 2-item tuple, the first item will be the value to the
480 left of the separator and the second item will be the value to the
481 right. By default `dict`.
482 default_key : `str` or `None`
483 The key to use if a value is passed that is not a key-value pair.
484 `None` can imply no separator depending on how the results are handled.
485 (Passing values that are not key-value pairs requires
486 ``unseparated_okay`` to be `True`).
487 reverse_kv : bool
488 If true then for each item in values, the value to the left of the
489 separator is treated as the value and the value to the right of the
490 separator is treated as the key. By default `False`.
491 add_to_default : `bool`, optional
492 If True, then passed-in values will not overwrite the default value
493 unless the ``return_type`` is `dict` and passed-in value(s) have the
494 same key(s) as the default value.
496 Returns
497 -------
498 values : `dict` [`str`, `str`] or `tuple` [`tuple` [`str`, `str`], ...]
499 The passed-in values in dict form or tuple form.
501 Raises
502 ------
503 `click.ClickException`
504 Raised if the separator is not found in an entry, or if duplicate keys
505 are encountered.
506 """
508 def norm(val: str) -> str:
509 """If `normalize` is `True` and `choice` is not `None`, find the value
510 in the available choices and return the value as spelled in the
511 choices.
513 Assumes that val exists in choices; `split_kv` uses the `choice`
514 instance to verify val is a valid choice.
516 Parameters
517 ----------
518 val : `str`
519 Value to be found.
521 Returns
522 -------
523 val : `str`
524 The value that was found or the value that was given.
525 """
526 if normalize and choice is not None:
527 v = val.casefold()
528 for opt in choice.choices:
529 if opt.casefold() == v:
530 return opt
531 return val
533 class RetDict:
534 def __init__(self) -> None:
535 self.ret: dict[str | None, str] = {}
537 def add(self, key: str | None, val: str) -> None:
538 if reverse_kv:
539 key, val = val, key
540 self.ret[key] = val
542 def get(self) -> dict[str | None, str]:
543 return self.ret
545 class RetTuple:
546 def __init__(self) -> None:
547 self.ret: list[tuple[str | None, str]] = []
549 def add(self, key: str | None, val: str) -> None:
550 if reverse_kv:
551 key, val = val, key
552 self.ret.append((key, val))
554 def get(self) -> tuple[tuple[str | None, str], ...]:
555 return tuple(self.ret)
557 if separator in (",", " "):
558 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
559 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
561 if add_to_default:
562 default = param.get_default(context)
563 if default and default != _CLICK_UNSET_SENTINEL:
564 vals = tuple(
565 v for v in itertools.chain(ensure_iterable(default), vals)
566 ) # Convert to tuple for mypy
568 ret: RetDict | RetTuple
569 if return_type is dict:
570 ret = RetDict()
571 elif return_type is tuple:
572 ret = RetTuple()
573 else:
574 raise click.ClickException(
575 message=f"Internal error: invalid return type '{return_type}' for split_kv."
576 )
577 if multiple:
578 vals = split_commas(context, param, vals)
579 for val in ensure_iterable(vals):
580 if unseparated_okay and separator not in val:
581 if choice is not None:
582 choice(val) # will raise if val is an invalid choice
583 ret.add(default_key, norm(val))
584 else:
585 try:
586 k, v = val.split(separator)
587 if choice is not None:
588 choice(v) # will raise if val is an invalid choice
589 except ValueError as e:
590 raise click.ClickException(
591 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
592 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
593 ) from None
594 ret.add(k, norm(v))
595 return ret.get()
598def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
599 """Convert a value to upper case.
601 Parameters
602 ----------
603 context : `click.Context`
604 Context given by Click.
605 param : `click.core.Option`
606 Provided by Click. Ignored.
607 value : `str`
608 The value to be converted.
610 Returns
611 -------
612 str
613 A copy of the passed-in value, converted to upper case.
614 """
615 return value.upper()
618def unwrap(val: str) -> str:
619 """Remove newlines and leading whitespace from a multi-line string with
620 a consistent indentation level.
622 The first line of the string may be only a newline or may contain text
623 followed by a newline, either is ok. After the first line, each line must
624 begin with a consistant amount of whitespace. So, content of a
625 triple-quoted string may begin immediately after the quotes, or the string
626 may start with a newline. Each line after that must be the same amount of
627 indentation/whitespace followed by text and a newline. The last line may
628 end with a new line but is not required to do so.
630 Parameters
631 ----------
632 val : `str`
633 The string to change.
635 Returns
636 -------
637 strippedString : `str`
638 The string with newlines, indentation, and leading and trailing
639 whitespace removed.
640 """
642 def splitSection(val: str) -> str:
643 if not val.startswith("\n"): 643 ↛ 647line 643 didn't jump to line 647 because the condition on line 643 was always true
644 firstLine, _, val = val.partition("\n")
645 firstLine += " "
646 else:
647 firstLine = ""
648 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
650 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
653class option_section: # noqa: N801
654 """Decorator to add a section label between options in the help text of a
655 command.
657 Parameters
658 ----------
659 sectionText : `str`
660 The text to print in the section identifier.
661 """
663 def __init__(self, sectionText: str) -> None:
664 self.sectionText = "\n" + sectionText
666 def __call__(self, f: Any) -> click.Option:
667 # Generate a parameter declaration that will be unique for this
668 # section.
669 return click.option(
670 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
671 )(f)
674class MWPath(click.Path):
675 """Overrides `click.Path` to implement file-does-not-exist checking.
677 Changes the definition of ``exists`` so that `True` indicates the location
678 (file or directory) must exist, `False` indicates the location must *not*
679 exist, and `None` indicates that the file may exist or not. The standard
680 definition for the `click.Path` ``exists`` parameter is that for `True` a
681 location must exist, but `False` means it is not required to exist (not
682 that it is required to not exist).
684 Parameters
685 ----------
686 exists : `bool` or `None`, optional
687 If `True`, the location (file or directory) indicated by the caller
688 must exist. If `False` the location must not exist. If `None`, the
689 location may exist or not.
690 file_okay : `bool`, optional
691 Allow a file as a value.
692 dir_okay : `bool`, optional
693 Allow a directory as a value.
694 writable : `bool`, optional
695 If `True`, a writable check is performed.
696 readable : `bool`, optional
697 If `True`, a readable check is performed.
698 resolve_path : `bool`, optional
699 Resolve the path.
700 allow_dash : `bool`, optional
701 Allow single dash as value to mean a standard stream.
702 path_type : `type` or `None`, optional
703 Convert the incoming value to this type.
705 Notes
706 -----
707 All parameters other than ``exists`` come directly from `click.Path`.
708 """
710 def __init__(
711 self,
712 exists: bool | None = None,
713 file_okay: bool = True,
714 dir_okay: bool = True,
715 writable: bool = False,
716 readable: bool = True,
717 resolve_path: bool = False,
718 allow_dash: bool = False,
719 path_type: type | None = None,
720 ):
721 self.mustNotExist = exists is False
722 if exists is None: 722 ↛ 724line 722 didn't jump to line 724 because the condition on line 722 was always true
723 exists = False
724 super().__init__(
725 exists=exists,
726 file_okay=file_okay,
727 dir_okay=dir_okay,
728 writable=writable,
729 readable=readable,
730 resolve_path=resolve_path,
731 allow_dash=allow_dash,
732 path_type=path_type,
733 )
735 def convert(
736 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None
737 ) -> Any:
738 """Convert values through types.
740 Called by `click.ParamType` to "convert values through types".
741 `click.Path` uses this step to verify Path conditions.
743 Parameters
744 ----------
745 value : `str` or `os.PathLike`
746 File path.
747 param : `click.Parameter`
748 Parameters provided by Click.
749 ctx : `click.Context`
750 Context provided by Click.
751 """
752 if self.mustNotExist and os.path.exists(value):
753 self.fail(f'Path "{value}" should not exist.')
754 return super().convert(value, param, ctx)
757class MWOption(click.Option):
758 """Overrides click.Option with desired behaviors."""
760 def make_metavar(self, ctx: click.Context | None = None) -> str:
761 """Make the metavar for the help menu.
763 Parameters
764 ----------
765 ctx : `click.Context` or `None`
766 Context from the command.
768 Notes
769 -----
770 Overrides `click.Option.make_metavar`.
771 Adds a space and an ellipsis after the metavar name if
772 the option accepts multiple inputs, otherwise defers to the base
773 implementation.
775 By default click does not add an ellipsis when multiple is True and
776 nargs is 1. And when nargs does not equal 1 click adds an ellipsis
777 without a space between the metavar and the ellipsis, but we prefer a
778 space between.
780 Does not get called for some option types (e.g. flag) so metavar
781 transformation that must apply to all types should be applied in
782 get_help_record.
783 """
784 if _click_make_metavar_has_context:
785 metavar = super().make_metavar(ctx=ctx) # type: ignore
786 else:
787 metavar = super().make_metavar() # type: ignore
788 if self.multiple and self.nargs == 1:
789 metavar += " ..."
790 elif self.nargs != 1:
791 metavar = f"{metavar[:-3]} ..."
792 return metavar
795class MWArgument(click.Argument):
796 """Overrides click.Argument with desired behaviors."""
798 def make_metavar(self, ctx: click.Context | None = None) -> str:
799 """Make the metavar for the help menu.
801 Parameters
802 ----------
803 ctx : `click.Context` or `None`
804 Context from the command.
806 Notes
807 -----
808 Overrides `click.Option.make_metavar`.
809 Always adds a space and an ellipsis (' ...') after the
810 metavar name if the option accepts multiple inputs.
812 By default click adds an ellipsis without a space between the metavar
813 and the ellipsis, but we prefer a space between.
815 Returns
816 -------
817 metavar : `str`
818 The metavar value.
819 """
820 if _click_make_metavar_has_context:
821 metavar = super().make_metavar(ctx=ctx) # type: ignore
822 else:
823 metavar = super().make_metavar() # type: ignore
824 if self.nargs != 1:
825 metavar = f"{metavar[:-3]} ..."
826 return metavar
829class OptionSection(MWOption):
830 """Implements an Option that prints a section label in the help text and
831 does not pass any value to the command function.
833 This class does a bit of hackery to add a section label to a click command
834 help output: first, ``expose_value`` is set to `False` so that no value is
835 passed to the command function. Second, this class overrides
836 `click.Option.get_help_record` to return the section label string without
837 any prefix so that it stands out as a section label.
839 This class overrides the hidden attribute because our documentation build
840 tool, sphinx-click, implements its own `get_help_record` function which
841 builds the record from other option values (e.g. ``name``, ``opts``), which
842 breaks the hack we use to make `get_help_record` only return the
843 ``sectionText``. Fortunately, Click gets the value of `hidden` inside the
844 `click.Option`'s `get_help_record`, and sphinx-click calls ``opt.hidden``
845 before entering its ``_get_help_record`` function. So, making the hidden
846 property return True hides this option from sphinx-click, while allowing
847 the section text to be returned by our `get_help_record` method when using
848 Click.
850 The intention for this implementation is to do minimally invasive overrides
851 of the click classes so as to be robust and easy to fix if the click
852 internals change.
854 Parameters
855 ----------
856 sectionName : `str`
857 The parameter declaration for this option. It is not shown to the user,
858 it must be unique within the command. If using the ``section``
859 decorator to add a section to a command's options, the section name is
860 auto-generated.
861 sectionText : `str`
862 The text to print in the section identifier.
863 """
865 @property
866 def hidden(self) -> bool:
867 return True
869 @hidden.setter
870 def hidden(self, val: Any) -> None:
871 pass
873 def __init__(self, sectionName: str, sectionText: str) -> None:
874 super().__init__(sectionName, expose_value=False)
875 self.sectionText = sectionText
877 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
878 return (self.sectionText, "")
881class MWOptionDecorator:
882 """Wraps the click.option decorator to enable shared options to be declared
883 and allows inspection of the shared option.
885 Parameters
886 ----------
887 *param_decls : `typing.Any`
888 Parameters to be stored in the option.
889 **kwargs : `typing.Any`
890 Keyword arguments for the option.
891 """
893 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
894 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) # type: ignore
895 opt = click.Option(param_decls, **kwargs)
896 self._name = opt.name
897 self._opts = opt.opts
899 def name(self) -> str:
900 """Get the name that will be passed to the command function for this
901 option.
902 """
903 return cast(str, self._name)
905 def opts(self) -> list[str]:
906 """Get the flags that will be used for this option on the command
907 line.
908 """
909 return self._opts
911 @property
912 def help(self) -> str:
913 """Get the help text for this option. Returns an empty string if no
914 help was defined.
915 """
916 return self.partialOpt.keywords.get("help", "")
918 def __call__(self, *args: Any, **kwargs: Any) -> Any:
919 return self.partialOpt(*args, **kwargs)
922class MWArgumentDecorator:
923 """Wraps the click.argument decorator to enable shared arguments to be
924 declared.
926 Parameters
927 ----------
928 *param_decls : `typing.Any`
929 Parameters to be stored in the argument.
930 **kwargs : `typing.Any`
931 Keyword arguments for the argument.
932 """
934 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
935 self._helpText = kwargs.pop("help", None)
936 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
938 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
939 def decorator(f: Any) -> Any:
940 if help is not None:
941 self._helpText = help
942 if self._helpText:
943 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
944 return self.partialArg(*args, **kwargs)(f)
946 return decorator
949class MWCommand(click.Command):
950 """Command subclass that stores a copy of the args list for use by the
951 command.
953 Parameters
954 ----------
955 *args : `typing.Any`
956 Arguments for `click.Command`.
957 **kwargs : `typing.Any`
958 Keyword arguments for `click.Command`.
959 """
961 name = "butler"
962 extra_epilog: str | None = None
964 def __init__(self, *args: Any, **kwargs: Any) -> None:
965 # wrap callback method with catch_and_exit decorator
966 callback = kwargs.get("callback")
967 if callback is not None: 967 ↛ 970line 967 didn't jump to line 970 because the condition on line 967 was always true
968 kwargs = kwargs.copy()
969 kwargs["callback"] = catch_and_exit(callback)
970 super().__init__(*args, **kwargs)
972 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
973 """Capture the command line options and arguments.
975 See details about what is captured and the order in which it is stored
976 in the documentation of `MWCtxObj`.
978 Parameters
979 ----------
980 ctx : `click.Context`
981 The current Context.
982 args : `list` [`str`]
983 The list of arguments from the command line, split at spaces but
984 not at separators (like "=").
985 """
986 parser = self.make_parser(ctx)
987 opts, _, param_order = parser.parse_args(args=list(args))
988 # `param_order` is a list of click.Option and click.Argument, there is
989 # one item for each time the Option or Argument was used on the
990 # command line. Options will precede Arguments, within each sublist
991 # they are in the order they were used on the command line. Note that
992 # click.Option and click.Argument do not contain the value from the
993 # command line; values are in `opts`.
994 #
995 # `opts` is a dict where the key is the argument name to the
996 # click.Command function, this name matches the `click.Option.name` or
997 # `click.Argument.name`. For Options, an item will only be present if
998 # the Option was used on the command line. For Arguments, an item will
999 # always be present and if no value was provided on the command line
1000 # the value will be `None`. If the option accepts multiple values, the
1001 # value in `opts` is a tuple, otherwise it is a single item.
1002 next_idx: Counter = Counter()
1003 captured_args = []
1004 for param in param_order:
1005 if isinstance(param, click.Option):
1006 param_name = cast(str, param.name)
1007 if param.multiple:
1008 val = opts[param_name][next_idx[param_name]]
1009 next_idx[param_name] += 1
1010 else:
1011 val = opts[param_name]
1012 if param.is_flag:
1013 # Bool options store their True flags in opts and their
1014 # False flags in secondary_opts.
1015 if val:
1016 flag = max(param.opts, key=len)
1017 else:
1018 flag = max(param.secondary_opts, key=len)
1019 captured_args.append(flag)
1020 else:
1021 captured_args.append(max(param.opts, key=len))
1022 captured_args.append(val)
1023 elif isinstance(param, click.Argument):
1024 param_name = cast(str, param.name)
1025 opt = opts[param_name]
1026 if opt is not None and opt != _CLICK_UNSET_SENTINEL:
1027 captured_args.append(opt)
1028 else:
1029 raise AssertionError("All parameters should be an Option or an Argument")
1030 MWCtxObj.getFrom(ctx).args = captured_args
1032 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
1033 """Given a context and a list of arguments this creates the parser and
1034 parses the arguments, then modifies the context as necessary. This is
1035 automatically invoked by make_context().
1037 This function overrides `click.Command.parse_args`.
1039 The call to `_capture_args` in this override stores the arguments
1040 (option names, option value, and argument values) that were used by the
1041 caller on the command line in the context object. These stored
1042 arguments can be used by the command function, e.g. to process options
1043 in the order they appeared on the command line (``pipetask`` uses this
1044 feature to create pipeline actions in an order from different options).
1046 Parameters
1047 ----------
1048 ctx : `click.core.Context`
1049 The current Context.
1050 args : `list` [`str`]
1051 The list of arguments from the command line, split at spaces but
1052 not at separators (like "=").
1053 """
1054 self._capture_args(ctx, args)
1055 return super().parse_args(ctx, args)
1057 @property
1058 def epilog(self) -> str | None:
1059 """Override the epilog attribute to add extra_epilog (if defined by a
1060 subclass) to the end of any epilog provided by a subcommand.
1061 """
1062 ret = self._epilog if self._epilog else ""
1063 if self.extra_epilog:
1064 if ret:
1065 ret += "\n\n"
1066 ret += self.extra_epilog
1067 return ret
1069 @epilog.setter
1070 def epilog(self, val: str | None) -> None:
1071 self._epilog = val
1074class ButlerCommand(MWCommand):
1075 """Command subclass with butler-command specific overrides."""
1077 extra_epilog = "See 'butler --help' for more options."
1080class OptionGroup:
1081 """Base class for an option group decorator. Requires the option group
1082 subclass to have a property called ``decorator``.
1083 """
1085 decorators: list[Any]
1087 def __call__(self, f: Any) -> Any:
1088 for decorator in reversed(self.decorators):
1089 f = decorator(f)
1090 return f
1093class MWCtxObj:
1094 """Helper object for managing the `click.Context.obj` parameter, allows
1095 obj data to be managed in a consistent way.
1097 `click.Context.obj` defaults to None. ``MWCtxObj.getFrom(ctx)`` can be used
1098 to initialize the obj if needed and return a new or existing `MWCtxObj`.
1100 The `args` attribute contains a list of options, option values, and
1101 argument values that is similar to the list of arguments and options that
1102 were passed in on the command line, with differences noted below:
1104 * Option names and option values are first in the list, and argument
1105 values come last. The order of options and option values is preserved
1106 within the options. The order of argument values is preserved.
1108 * The longest option name is used for the option in the `args` list, e.g.
1109 if an option accepts both short and long names ``"-o / --option"`` and
1110 the short option name ``"-o"`` was used on the command line, the longer
1111 name will be the one that appears in ``args``.
1113 * A long option name (which begins with two dashes ``"--"``) and its value
1114 may be separated by an equal sign; the name and value are split at the
1115 equal sign and it is removed. In ``args``, the option is in one list
1116 item, and the option value (without the equal sign) is in the next list
1117 item. e.g. ``"--option=foo"`` and ``"--option foo"`` both become
1118 ``["--opt", "foo"]`` in ``args``.
1120 * A short option name, (which begins with one dash ``"-"``) and its value
1121 are split immediately after the short option name, and if there is
1122 whitespace between the short option name and its value it is removed.
1123 Everything after the short option name (excluding whitespace) is included
1124 in the value. If the ``Option`` has a long name, the long name will be
1125 used in ``args`` e.g. for the option ``"-o / --option"``: ``"-ofoo"`` and
1126 ``"-o foo"`` become ``["--option", "foo"]``, and (note!) ``"-o=foo"``
1127 will become ``["--option", "=foo"]`` (because everything after the short
1128 option name, except whitespace, is used for the value (as is standard
1129 with unix command line tools).
1131 Attributes
1132 ----------
1133 args : `list` [`str`]
1134 A list of options, option values, and arguments similar to those that
1135 were passed in on the command line. See comments about captured options
1136 & arguments above.
1137 """
1139 def __init__(self) -> None:
1140 self.args = None
1142 @staticmethod
1143 def getFrom(ctx: click.Context) -> Any:
1144 """If needed, initialize ``ctx.obj`` with a new `MWCtxObj`, and return
1145 the new or already existing `MWCtxObj`.
1147 Parameters
1148 ----------
1149 ctx : `click.Context`
1150 Context provided by Click.
1151 """
1152 if ctx.obj is not None:
1153 return ctx.obj
1154 ctx.obj = MWCtxObj()
1155 return ctx.obj
1158def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
1159 """Read additional values from the supplied YAML file.
1161 Parameters
1162 ----------
1163 ctx : `click.Context`
1164 The context for the click operation. Used to extract the subcommand
1165 name and translate option & argument names.
1166 param : `str`
1167 The parameter name.
1168 value : `object`
1169 The value of the parameter.
1170 """
1172 def _name_for_option(ctx: click.Context, option: str) -> str:
1173 """Use a CLI option name to find the name of the argument to the
1174 command function.
1176 Parameters
1177 ----------
1178 ctx : `click.Context`
1179 The context for the click operation.
1180 option : `str`
1181 The option/argument name from the yaml file.
1183 Returns
1184 -------
1185 name : str
1186 The name of the argument to use when calling the click.command
1187 function, as it should appear in the `ctx.default_map`.
1189 Raises
1190 ------
1191 RuntimeError
1192 Raised if the option name from the yaml file does not exist in the
1193 command parameters. This catches misspellings and incorrect usage
1194 in the yaml file.
1195 """
1196 for param in ctx.command.params:
1197 # Remove leading dashes: they are not used for option names in the
1198 # yaml file.
1199 if option in [opt.lstrip("-") for opt in param.opts]:
1200 return cast(str, param.name)
1201 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
1203 ctx.default_map = ctx.default_map or {}
1204 cmd_name = ctx.info_name
1205 assert cmd_name is not None, "command name cannot be None"
1206 if value:
1207 try:
1208 overrides = _read_yaml_presets(value, cmd_name)
1209 options = list(overrides.keys())
1210 for option in options:
1211 name = _name_for_option(ctx, option)
1212 if name == option:
1213 continue
1214 overrides[name] = overrides.pop(option)
1215 except Exception as e:
1216 raise click.BadOptionUsage(
1217 option_name=param,
1218 message=f"Error reading overrides file: {e}",
1219 ctx=ctx,
1220 ) from None
1221 # Override the defaults for this subcommand
1222 ctx.default_map.update(overrides)
1223 return
1226def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]:
1227 """Read file command line overrides from YAML config file.
1229 Parameters
1230 ----------
1231 file_uri : `str`
1232 URI of override YAML file containing the command line overrides.
1233 They should be grouped by command name.
1234 cmd_name : `str`
1235 The subcommand name that is being modified.
1237 Returns
1238 -------
1239 overrides : `dict` of [`str`, Any]
1240 The relevant command line options read from the override file.
1241 """
1242 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1243 config = Config(file_uri)
1244 return config[cmd_name]
1247def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1248 """Sort an astropy table.
1250 Prioritization is given to columns in this order:
1252 1. the provided named columns
1253 2. spatial and temporal columns
1254 3. the rest of the columns.
1256 The table is sorted in-place, and is also returned for convenience.
1258 Parameters
1259 ----------
1260 table : `astropy.table.Table`
1261 The table to sort.
1262 dimensions : `list` [``Dimension``]
1263 The dimensions of the dataIds in the table (the dimensions should be
1264 the same for all the dataIds). Used to determine if the column is
1265 spatial, temporal, or neither.
1266 sort_first : `list` [`str`]
1267 The names of columns that should be sorted first, before spatial and
1268 temporal columns.
1270 Returns
1271 -------
1272 `astropy.table.Table`
1273 For convenience, the table that has been sorted.
1274 """
1275 # For sorting we want to ignore the id
1276 # We also want to move temporal or spatial dimensions earlier
1277 sort_first = sort_first or []
1278 sort_early: list[str] = []
1279 sort_late: list[str] = []
1280 for dim in dimensions:
1281 if dim.spatial or dim.temporal:
1282 sort_early.extend(dim.required.names)
1283 else:
1284 sort_late.append(str(dim))
1285 sort_keys = sort_first + sort_early + sort_late
1286 # The required names above means that we have the possibility of
1287 # repeats of sort keys. Now have to remove them
1288 # (order is retained by dict creation).
1289 sort_keys = list(dict.fromkeys(sort_keys).keys())
1291 table.sort(sort_keys)
1292 return table
1295def catch_and_exit(func: Callable) -> Callable:
1296 """Catch all exceptions, prints an exception traceback
1297 and signals click to exit.
1299 Use as decorator.
1301 Parameters
1302 ----------
1303 func : `collections.abc.Callable`
1304 The function to be decorated.
1306 Returns
1307 -------
1308 `collections.abc.Callable`
1309 The decorated function.
1310 """
1312 @wraps(func)
1313 def inner(*args: Any, **kwargs: Any) -> None:
1314 try:
1315 func(*args, **kwargs)
1316 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1317 # this is handled by click itself
1318 raise
1319 except Exception:
1320 exc_type, exc_value, exc_tb = sys.exc_info()
1321 assert exc_type is not None
1322 assert exc_value is not None
1323 assert exc_tb is not None
1324 exit_hdl = ClickExitFailedNicely(exc_type, exc_value, exc_tb)
1325 exit_hdl.exit_click()
1327 return inner