Coverage for python / lsst / daf / butler / cli / utils.py: 33%
389 statements
« prev ^ index » next coverage.py v7.12.0, created at 2025-12-12 01:12 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This software is dual licensed under the GNU General Public License and also
10# under a 3-clause BSD license. Recipients may choose which of these licenses
11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt,
12# respectively. If you choose the GPL option then the following text applies
13# (but note that there is still no warranty even if you opt for BSD instead):
14#
15# This program is free software: you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation, either version 3 of the License, or
18# (at your option) any later version.
19#
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
24#
25# You should have received a copy of the GNU General Public License
26# along with this program. If not, see <http://www.gnu.org/licenses/>.
27from __future__ import annotations
29__all__ = (
30 "ButlerCommand",
31 "LogCliRunner",
32 "MWArgument",
33 "MWArgumentDecorator",
34 "MWCommand",
35 "MWCtxObj",
36 "MWOption",
37 "MWOptionDecorator",
38 "MWPath",
39 "OptionGroup",
40 "OptionSection",
41 "addArgumentHelp",
42 "astropyTablesToStr",
43 "catch_and_exit",
44 "clickResultMsg",
45 "command_test_env",
46 "option_section",
47 "printAstropyTables",
48 "sortAstropyTable",
49 "split_commas",
50 "split_kv",
51 "textTypeStr",
52 "to_upper",
53 "unwrap",
54 "yaml_presets",
55)
58import importlib.metadata
59import itertools
60import logging
61import os
62import re
63import sys
64import textwrap
65import traceback
66import types
67import uuid
68import warnings
69from collections import Counter
70from collections.abc import Callable, Iterable, Iterator
71from contextlib import contextmanager
72from functools import partial, wraps
73from typing import TYPE_CHECKING, Any, cast
74from unittest.mock import patch
76import click
77import click.core
78import click.exceptions
79import click.testing
80import yaml
81from packaging.version import Version
83from lsst.utils.iteration import ensure_iterable
85from .._config import Config
86from .cliLog import CliLog
88if TYPE_CHECKING:
89 from astropy.table import Table
91 from lsst.daf.butler import Dimension
93_click_version = Version(importlib.metadata.version("click"))
94if _click_version >= Version("8.2.0"): 94 ↛ 97line 94 didn't jump to line 97 because the condition on line 94 was always true
95 _click_make_metavar_has_context = True
96else:
97 _click_make_metavar_has_context = False
99# Starting from Click 8.3.0, a special `UNSET` sentinel value is used to
100# indicate the absence of a default value for a parameter. Prior to 8.3.0,
101# they just used `None`.
102_CLICK_UNSET_SENTINEL = getattr(click.core, "UNSET", None)
104log = logging.getLogger(__name__)
106# This is used as the metavar argument to Options that accept multiple string
107# inputs, which may be comma-separarated. For example:
108# --my-opt foo,bar --my-opt baz.
109# Other arguments to the Option should include multiple=true and
110# callback=split_kv.
111typeStrAcceptsMultiple = "TEXT ..."
112typeStrAcceptsSingle = "TEXT"
114# The standard help string for the --where option when it takes a WHERE clause.
115where_help = (
116 "A string expression similar to a SQL WHERE clause. May involve any column of a "
117 "dimension table or a dimension name as a shortcut for the primary key column of a "
118 "dimension table."
119)
122def astropyTablesToStr(tables: list[Table]) -> str:
123 """Render astropy tables to string as they are displayed in the CLI.
125 Output formatting matches ``printAstropyTables``.
127 Parameters
128 ----------
129 tables : `list` of `astropy.table.Table`
130 The tables to format.
132 Returns
133 -------
134 formatted : `str`
135 Tables formatted into a string.
136 """
137 ret = ""
138 for table in tables:
139 ret += "\n"
140 table.pformat()
141 ret += "\n"
142 return ret
145def printAstropyTables(tables: list[Table]) -> None:
146 """Print astropy tables to be displayed in the CLI.
148 Output formatting matches ``astropyTablesToStr``.
150 Parameters
151 ----------
152 tables : `list` of `astropy.table.Table`
153 The tables to print.
154 """
155 for table in tables:
156 print("")
157 table.pprint_all()
158 print("")
161def textTypeStr(multiple: bool) -> str:
162 """Get the text type string for CLI help documentation.
164 Parameters
165 ----------
166 multiple : `bool`
167 True if multiple text values are allowed, False if only one value is
168 allowed.
170 Returns
171 -------
172 textTypeStr : `str`
173 The type string to use.
174 """
175 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
178class ClickExitFailedNicely:
179 """Exit a Click command that failed.
181 This class is used to control the behavior when a click command has failed.
182 By default the exception will be logged and a non-zero exit status will
183 be used.
185 Parameters
186 ----------
187 exc_type : `type`
188 The type of the exception.
189 exc_value : `Exception`
190 The `Exception` object.
191 exc_tb : `types.TracebackType`
192 The traceback for this exception.
193 """
195 use_bad_status: bool = True
196 """Control how a bad command is exited. `True` indicates bad status and
197 `False` indicates a `click.ClickException`."""
199 def __init__(self, exc_type: type[BaseException], exc_value: BaseException, exc_tb: types.TracebackType):
200 self.exc_type = exc_type
201 self.exc_value = exc_value
202 self.exc_tb = self._clean_tb(exc_tb)
204 def _clean_tb(self, exc_tb: types.TracebackType) -> types.TracebackType:
205 if exc_tb.tb_next:
206 # Do not show the decorator in traceback.
207 exc_tb = exc_tb.tb_next
208 return exc_tb
210 def exit_click(self) -> None:
211 if self.use_bad_status:
212 self.exit_click_command_bad_status()
213 else:
214 self.exit_click_command_click_exception()
216 def exit_click_command_bad_status(self) -> None:
217 """Exit a click command with bad exit status and report log message."""
218 log.exception(
219 "Caught an exception, details are in traceback:",
220 exc_info=(self.exc_type, self.exc_value, self.exc_tb),
221 )
222 # Tell click to stop, this never returns.
223 click.get_current_context().exit(1)
225 def exit_click_command_click_exception(self) -> None:
226 """Exit a click command raising ClickException."""
227 tb = traceback.format_tb(self.exc_tb)
228 errmsg = "".join(tb) + str(self.exc_value)
229 raise click.ClickException(errmsg)
232class LogCliRunner(click.testing.CliRunner):
233 """A test runner to use when the logging system will be initialized by code
234 under test, calls CliLog.resetLog(), which undoes any logging setup that
235 was done with the CliLog interface.
237 lsst.log modules can not be set back to an uninitialized state (python
238 logging modules can be set back to NOTSET), instead they are set to
239 `CliLog.defaultLsstLogLevel`.
240 """
242 def invoke(self, *args: Any, **kwargs: Any) -> click.testing.Result:
243 # We want exceptions to be reported to the test runner rather than
244 # being converted to a simple exit status. The default is to
245 # use a logger but the click test infrastructure doesn't capture that
246 # in result.
247 with patch.object(ClickExitFailedNicely, "use_bad_status", False):
248 result = super().invoke(*args, **kwargs)
249 CliLog.resetLog()
250 if result.exception:
251 print("Failing command was: ", args)
252 return result
255def clickResultMsg(result: click.testing.Result) -> str:
256 """Get a standard assert message from a click result.
258 Parameters
259 ----------
260 result : click.testing.Result
261 The result object returned from `click.testing.CliRunner.invoke`.
263 Returns
264 -------
265 msg : `str`
266 The message string.
267 """
268 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
269 if result.exception:
270 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
271 return msg
274@contextmanager
275def command_test_env(runner: click.testing.CliRunner, commandModule: str, commandName: str) -> Iterator[None]:
276 """Context manager that creates (and then cleans up) an environment that
277 provides a CLI plugin command with the given name.
279 Parameters
280 ----------
281 runner : click.testing.CliRunner
282 The test runner to use to create the isolated filesystem.
283 commandModule : `str`
284 The importable module that the command can be imported from.
285 commandName : `str`
286 The name of the command being published to import.
287 """
288 with runner.isolated_filesystem():
289 with open("resources.yaml", "w") as f:
290 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
291 # Add a colon to the end of the path on the next line, this tests the
292 # case where the lookup in LoaderCLI._getPluginList generates an empty
293 # string in one of the list entries and verifies that the empty string
294 # is properly stripped out.
295 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
296 yield
299def addArgumentHelp(doc: str | None, helpText: str) -> str:
300 """Add a Click argument's help message to a function's documentation.
302 This is needed because click presents arguments in the order the argument
303 decorators are applied to a function, top down. But, the evaluation of the
304 decorators happens bottom up, so if arguments just append their help to the
305 function's docstring, the argument descriptions appear in reverse order
306 from the order they are applied in.
308 Parameters
309 ----------
310 doc : `str`
311 The function's docstring.
312 helpText : `str`
313 The argument's help string to be inserted into the function's
314 docstring.
316 Returns
317 -------
318 doc : `str`
319 Updated function documentation.
320 """
321 if doc is None: 321 ↛ 322line 321 didn't jump to line 322 because the condition on line 321 was never true
322 doc = helpText
323 else:
324 # See click documentation for details:
325 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
326 # In short, text for the click command help can be truncated by putting
327 # "\f" in the docstring, everything after it should be removed
328 if "\f" in doc: 328 ↛ 329line 328 didn't jump to line 329 because the condition on line 328 was never true
329 doc = doc.split("\f")[0]
331 doclines = doc.splitlines()
332 # The function's docstring may span multiple lines, so combine the
333 # docstring from all the first lines until a blank line is encountered.
334 # (Lines after the first blank line will be argument help.)
335 while len(doclines) > 1 and doclines[1]:
336 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
337 # Add standard indent to help text for proper alignment with command
338 # function documentation:
339 helpText = " " + helpText
340 doclines.insert(1, helpText)
341 doclines.insert(1, "\n")
342 doc = "\n".join(doclines)
343 return doc
346def split_commas(
347 context: click.Context | None, param: click.core.Option | None, values: str | Iterable[str] | None
348) -> tuple[str, ...]:
349 """Process a tuple of values, where each value may contain comma-separated
350 values, and return a single list of all the passed-in values.
352 This function can be passed to the 'callback' argument of a click.option to
353 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
354 the comma is inside ``[]`` there will be no splitting.
356 Parameters
357 ----------
358 context : `click.Context` or `None`
359 The current execution context. Unused, but Click always passes it to
360 callbacks.
361 param : `click.core.Option` or `None`
362 The parameter being handled. Unused, but Click always passes it to
363 callbacks.
364 values : iterable of `str` or `str`
365 All the values passed for this option. Strings may contain commas,
366 which will be treated as delimiters for separate values unless they
367 are within ``[]``.
369 Returns
370 -------
371 results : `tuple` [`str`]
372 The passed in values separated by commas where appropriate and
373 combined into a single tuple.
374 """
375 if values is None:
376 return ()
377 valueList = []
378 for value in ensure_iterable(values):
379 # If we have [, or ,] we do the slow split. If square brackets
380 # are not matching then that is likely a typo that should result
381 # in a warning.
382 opens = "["
383 closes = "]"
384 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
385 in_parens = False
386 current = ""
387 for c in value:
388 if c == opens:
389 if in_parens:
390 warnings.warn(
391 f"Found second opening {opens} without corresponding closing {closes}"
392 f" in {value!r}",
393 stacklevel=2,
394 )
395 in_parens = True
396 elif c == closes:
397 if not in_parens:
398 warnings.warn(
399 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
400 stacklevel=2,
401 )
402 in_parens = False
403 elif c == "," and not in_parens:
404 # Split on this comma.
405 valueList.append(current)
406 current = ""
407 continue
408 current += c
409 if in_parens:
410 warnings.warn(
411 f"Found opening {opens} that was never closed in {value!r}",
412 stacklevel=2,
413 )
414 if current:
415 valueList.append(current)
416 else:
417 # Use efficient split since no parens.
418 valueList.extend(value.split(","))
419 return tuple(valueList)
422def split_kv(
423 context: click.Context,
424 param: click.core.Option,
425 values: list[str],
426 *,
427 choice: click.Choice | None = None,
428 multiple: bool = True,
429 normalize: bool = False,
430 separator: str = "=",
431 unseparated_okay: bool = False,
432 return_type: type[dict] | type[tuple] = dict,
433 default_key: str | None = "",
434 reverse_kv: bool = False,
435 add_to_default: bool = False,
436) -> dict[str | None, str] | tuple[tuple[str | None, str], ...]:
437 """Process a tuple of values that are key-value pairs separated by a given
438 separator. Multiple pairs may be comma separated. Return a dictionary of
439 all the passed-in values.
441 This function can be passed to the 'callback' argument of a click.option to
442 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
444 Parameters
445 ----------
446 context : `click.Context` or `None`
447 The current execution context. Unused, but Click always passes it to
448 callbacks.
449 param : `click.core.Option` or `None`
450 The parameter being handled. Unused, but Click always passes it to
451 callbacks.
452 values : [`str`]
453 All the values passed for this option. Strings may contain commas,
454 which will be treated as delimiters for separate values.
455 choice : `click.Choice`, optional
456 If provided, verify each value is a valid choice using the provided
457 `click.Choice` instance. If None, no verification will be done. By
458 default `None`.
459 multiple : `bool`, optional
460 If true, the value may contain multiple comma-separated values. By
461 default True.
462 normalize : `bool`, optional
463 If True and `choice.case_sensitive == False`, normalize the string the
464 user provided to match the choice's case. By default `False`.
465 separator : str, optional
466 The character that separates key-value pairs. May not be a comma or an
467 empty space (for space separators use Click's default implementation
468 for tuples; `type=(str, str)`). By default "=".
469 unseparated_okay : `bool`, optional
470 If True, allow values that do not have a separator. They will be
471 returned in the values dict as a tuple of values in the key '', that
472 is: `values[''] = (unseparated_values, )`. By default False.
473 return_type : `type`, must be `dict` or `tuple`
474 The type of the value that should be returned.
475 If `dict` then the returned object will be a dict, for each item in
476 values, the value to the left of the separator will be the key and the
477 value to the right of the separator will be the value.
478 If `tuple` then the returned object will be a tuple. Each item in the
479 tuple will be 2-item tuple, the first item will be the value to the
480 left of the separator and the second item will be the value to the
481 right. By default `dict`.
482 default_key : `str` or `None`
483 The key to use if a value is passed that is not a key-value pair.
484 `None` can imply no separator depending on how the results are handled.
485 (Passing values that are not key-value pairs requires
486 ``unseparated_okay`` to be `True`).
487 reverse_kv : bool
488 If true then for each item in values, the value to the left of the
489 separator is treated as the value and the value to the right of the
490 separator is treated as the key. By default `False`.
491 add_to_default : `bool`, optional
492 If True, then passed-in values will not overwrite the default value
493 unless the ``return_type`` is `dict` and passed-in value(s) have the
494 same key(s) as the default value.
496 Returns
497 -------
498 values : `dict` [`str`, `str`] or `tuple`[`tuple`[`str`, `str`], ...]
499 The passed-in values in dict form or tuple form.
501 Raises
502 ------
503 `click.ClickException`
504 Raised if the separator is not found in an entry, or if duplicate keys
505 are encountered.
506 """
508 def norm(val: str) -> str:
509 """If `normalize` is `True` and `choice` is not `None`, find the value
510 in the available choices and return the value as spelled in the
511 choices.
513 Assumes that val exists in choices; `split_kv` uses the `choice`
514 instance to verify val is a valid choice.
516 Parameters
517 ----------
518 val : `str`
519 Value to be found.
521 Returns
522 -------
523 val : `str`
524 The value that was found or the value that was given.
525 """
526 if normalize and choice is not None:
527 v = val.casefold()
528 for opt in choice.choices:
529 if opt.casefold() == v:
530 return opt
531 return val
533 class RetDict:
534 def __init__(self) -> None:
535 self.ret: dict[str | None, str] = {}
537 def add(self, key: str | None, val: str) -> None:
538 if reverse_kv:
539 key, val = val, key
540 self.ret[key] = val
542 def get(self) -> dict[str | None, str]:
543 return self.ret
545 class RetTuple:
546 def __init__(self) -> None:
547 self.ret: list[tuple[str | None, str]] = []
549 def add(self, key: str | None, val: str) -> None:
550 if reverse_kv:
551 key, val = val, key
552 self.ret.append((key, val))
554 def get(self) -> tuple[tuple[str | None, str], ...]:
555 return tuple(self.ret)
557 if separator in (",", " "):
558 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
559 vals = tuple(ensure_iterable(values)) # preserve the original argument for error reporting below.
561 if add_to_default:
562 default = param.get_default(context)
563 if default and default != _CLICK_UNSET_SENTINEL:
564 vals = tuple(v for v in itertools.chain(default, vals)) # Convert to tuple for mypy
566 ret: RetDict | RetTuple
567 if return_type is dict:
568 ret = RetDict()
569 elif return_type is tuple:
570 ret = RetTuple()
571 else:
572 raise click.ClickException(
573 message=f"Internal error: invalid return type '{return_type}' for split_kv."
574 )
575 if multiple:
576 vals = split_commas(context, param, vals)
577 for val in ensure_iterable(vals):
578 if unseparated_okay and separator not in val:
579 if choice is not None:
580 choice(val) # will raise if val is an invalid choice
581 ret.add(default_key, norm(val))
582 else:
583 try:
584 k, v = val.split(separator)
585 if choice is not None:
586 choice(v) # will raise if val is an invalid choice
587 except ValueError as e:
588 raise click.ClickException(
589 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
590 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
591 ) from None
592 ret.add(k, norm(v))
593 return ret.get()
596def to_upper(context: click.Context, param: click.core.Option, value: str) -> str:
597 """Convert a value to upper case.
599 Parameters
600 ----------
601 context : `click.Context`
602 Context given by Click.
603 param : `click.core.Option`
604 Provided by Click. Ignored.
605 value : `str`
606 The value to be converted.
608 Returns
609 -------
610 string
611 A copy of the passed-in value, converted to upper case.
612 """
613 return value.upper()
616def unwrap(val: str) -> str:
617 """Remove newlines and leading whitespace from a multi-line string with
618 a consistent indentation level.
620 The first line of the string may be only a newline or may contain text
621 followed by a newline, either is ok. After the first line, each line must
622 begin with a consistant amount of whitespace. So, content of a
623 triple-quoted string may begin immediately after the quotes, or the string
624 may start with a newline. Each line after that must be the same amount of
625 indentation/whitespace followed by text and a newline. The last line may
626 end with a new line but is not required to do so.
628 Parameters
629 ----------
630 val : `str`
631 The string to change.
633 Returns
634 -------
635 strippedString : `str`
636 The string with newlines, indentation, and leading and trailing
637 whitespace removed.
638 """
640 def splitSection(val: str) -> str:
641 if not val.startswith("\n"): 641 ↛ 645line 641 didn't jump to line 645 because the condition on line 641 was always true
642 firstLine, _, val = val.partition("\n")
643 firstLine += " "
644 else:
645 firstLine = ""
646 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
648 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
651class option_section: # noqa: N801
652 """Decorator to add a section label between options in the help text of a
653 command.
655 Parameters
656 ----------
657 sectionText : `str`
658 The text to print in the section identifier.
659 """
661 def __init__(self, sectionText: str) -> None:
662 self.sectionText = "\n" + sectionText
664 def __call__(self, f: Any) -> click.Option:
665 # Generate a parameter declaration that will be unique for this
666 # section.
667 return click.option(
668 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
669 )(f)
672class MWPath(click.Path):
673 """Overrides `click.Path` to implement file-does-not-exist checking.
675 Changes the definition of ``exists`` so that `True` indicates the location
676 (file or directory) must exist, `False` indicates the location must *not*
677 exist, and `None` indicates that the file may exist or not. The standard
678 definition for the `click.Path` ``exists`` parameter is that for `True` a
679 location must exist, but `False` means it is not required to exist (not
680 that it is required to not exist).
682 Parameters
683 ----------
684 exists : `bool` or `None`, optional
685 If `True`, the location (file or directory) indicated by the caller
686 must exist. If `False` the location must not exist. If `None`, the
687 location may exist or not.
688 file_okay : `bool`, optional
689 Allow a file as a value.
690 dir_okay : `bool`, optional
691 Allow a directory as a value.
692 writable : `bool`, optional
693 If `True`, a writable check is performed.
694 readable : `bool`, optional
695 If `True`, a readable check is performed.
696 resolve_path : `bool`, optional
697 Resolve the path.
698 allow_dash : `bool`, optional
699 Allow single dash as value to mean a standard stream.
700 path_type : `type` or `None`, optional
701 Convert the incoming value to this type.
703 Notes
704 -----
705 All parameters other than ``exists`` come directly from `click.Path`.
706 """
708 def __init__(
709 self,
710 exists: bool | None = None,
711 file_okay: bool = True,
712 dir_okay: bool = True,
713 writable: bool = False,
714 readable: bool = True,
715 resolve_path: bool = False,
716 allow_dash: bool = False,
717 path_type: type | None = None,
718 ):
719 self.mustNotExist = exists is False
720 if exists is None: 720 ↛ 722line 720 didn't jump to line 722 because the condition on line 720 was always true
721 exists = False
722 super().__init__(
723 exists=exists,
724 file_okay=file_okay,
725 dir_okay=dir_okay,
726 writable=writable,
727 readable=readable,
728 resolve_path=resolve_path,
729 allow_dash=allow_dash,
730 path_type=path_type,
731 )
733 def convert(
734 self, value: str | os.PathLike[str], param: click.Parameter | None, ctx: click.Context | None
735 ) -> Any:
736 """Convert values through types.
738 Called by `click.ParamType` to "convert values through types".
739 `click.Path` uses this step to verify Path conditions.
741 Parameters
742 ----------
743 value : `str` or `os.PathLike`
744 File path.
745 param : `click.Parameter`
746 Parameters provided by Click.
747 ctx : `click.Context`
748 Context provided by Click.
749 """
750 if self.mustNotExist and os.path.exists(value):
751 self.fail(f'Path "{value}" should not exist.')
752 return super().convert(value, param, ctx)
755class MWOption(click.Option):
756 """Overrides click.Option with desired behaviors."""
758 def make_metavar(self, ctx: click.Context | None = None) -> str:
759 """Make the metavar for the help menu.
761 Parameters
762 ----------
763 ctx : `click.Context` or `None`
764 Context from the command.
766 Notes
767 -----
768 Overrides `click.Option.make_metavar`.
769 Adds a space and an ellipsis after the metavar name if
770 the option accepts multiple inputs, otherwise defers to the base
771 implementation.
773 By default click does not add an ellipsis when multiple is True and
774 nargs is 1. And when nargs does not equal 1 click adds an ellipsis
775 without a space between the metavar and the ellipsis, but we prefer a
776 space between.
778 Does not get called for some option types (e.g. flag) so metavar
779 transformation that must apply to all types should be applied in
780 get_help_record.
781 """
782 if _click_make_metavar_has_context:
783 metavar = super().make_metavar(ctx=ctx) # type: ignore
784 else:
785 metavar = super().make_metavar() # type: ignore
786 if self.multiple and self.nargs == 1:
787 metavar += " ..."
788 elif self.nargs != 1:
789 metavar = f"{metavar[:-3]} ..."
790 return metavar
793class MWArgument(click.Argument):
794 """Overrides click.Argument with desired behaviors."""
796 def make_metavar(self, ctx: click.Context | None = None) -> str:
797 """Make the metavar for the help menu.
799 Parameters
800 ----------
801 ctx : `click.Context` or `None`
802 Context from the command.
804 Notes
805 -----
806 Overrides `click.Option.make_metavar`.
807 Always adds a space and an ellipsis (' ...') after the
808 metavar name if the option accepts multiple inputs.
810 By default click adds an ellipsis without a space between the metavar
811 and the ellipsis, but we prefer a space between.
813 Returns
814 -------
815 metavar : `str`
816 The metavar value.
817 """
818 if _click_make_metavar_has_context:
819 metavar = super().make_metavar(ctx=ctx) # type: ignore
820 else:
821 metavar = super().make_metavar() # type: ignore
822 if self.nargs != 1:
823 metavar = f"{metavar[:-3]} ..."
824 return metavar
827class OptionSection(MWOption):
828 """Implements an Option that prints a section label in the help text and
829 does not pass any value to the command function.
831 This class does a bit of hackery to add a section label to a click command
832 help output: first, `expose_value` is set to `False` so that no value is
833 passed to the command function. Second, this class overrides
834 `click.Option.get_help_record` to return the section label string without
835 any prefix so that it stands out as a section label.
837 This class overrides the hidden attribute because our documentation build
838 tool, sphinx-click, implements its own `get_help_record` function which
839 builds the record from other option values (e.g. `name`, `opts`), which
840 breaks the hack we use to make `get_help_record` only return the
841 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
842 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
843 entering its `_get_help_record` function. So, making the hidden property
844 return True hides this option from sphinx-click, while allowing the section
845 text to be returned by our `get_help_record` method when using Click.
847 The intention for this implementation is to do minimally invasive overrides
848 of the click classes so as to be robust and easy to fix if the click
849 internals change.
851 Parameters
852 ----------
853 sectionName : `str`
854 The parameter declaration for this option. It is not shown to the user,
855 it must be unique within the command. If using the `section` decorator
856 to add a section to a command's options, the section name is
857 auto-generated.
858 sectionText : `str`
859 The text to print in the section identifier.
860 """
862 @property
863 def hidden(self) -> bool:
864 return True
866 @hidden.setter
867 def hidden(self, val: Any) -> None:
868 pass
870 def __init__(self, sectionName: str, sectionText: str) -> None:
871 super().__init__(sectionName, expose_value=False)
872 self.sectionText = sectionText
874 def get_help_record(self, ctx: click.Context | None) -> tuple[str, str]:
875 return (self.sectionText, "")
878class MWOptionDecorator:
879 """Wraps the click.option decorator to enable shared options to be declared
880 and allows inspection of the shared option.
882 Parameters
883 ----------
884 *param_decls : `typing.Any`
885 Parameters to be stored in the option.
886 **kwargs : `typing.Any`
887 Keyword arguments for the option.
888 """
890 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
891 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs) # type: ignore
892 opt = click.Option(param_decls, **kwargs)
893 self._name = opt.name
894 self._opts = opt.opts
896 def name(self) -> str:
897 """Get the name that will be passed to the command function for this
898 option.
899 """
900 return cast(str, self._name)
902 def opts(self) -> list[str]:
903 """Get the flags that will be used for this option on the command
904 line.
905 """
906 return self._opts
908 @property
909 def help(self) -> str:
910 """Get the help text for this option. Returns an empty string if no
911 help was defined.
912 """
913 return self.partialOpt.keywords.get("help", "")
915 def __call__(self, *args: Any, **kwargs: Any) -> Any:
916 return self.partialOpt(*args, **kwargs)
919class MWArgumentDecorator:
920 """Wraps the click.argument decorator to enable shared arguments to be
921 declared.
923 Parameters
924 ----------
925 *param_decls : `typing.Any`
926 Parameters to be stored in the argument.
927 **kwargs : `typing.Any`
928 Keyword arguments for the argument.
929 """
931 def __init__(self, *param_decls: Any, **kwargs: Any) -> None:
932 self._helpText = kwargs.pop("help", None)
933 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
935 def __call__(self, *args: Any, help: str | None = None, **kwargs: Any) -> Callable:
936 def decorator(f: Any) -> Any:
937 if help is not None:
938 self._helpText = help
939 if self._helpText:
940 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
941 return self.partialArg(*args, **kwargs)(f)
943 return decorator
946class MWCommand(click.Command):
947 """Command subclass that stores a copy of the args list for use by the
948 command.
950 Parameters
951 ----------
952 *args : `typing.Any`
953 Arguments for `click.Command`.
954 **kwargs : `typing.Any`
955 Keyword arguments for `click.Command`.
956 """
958 name = "butler"
959 extra_epilog: str | None = None
961 def __init__(self, *args: Any, **kwargs: Any) -> None:
962 # wrap callback method with catch_and_exit decorator
963 callback = kwargs.get("callback")
964 if callback is not None: 964 ↛ 967line 964 didn't jump to line 967 because the condition on line 964 was always true
965 kwargs = kwargs.copy()
966 kwargs["callback"] = catch_and_exit(callback)
967 super().__init__(*args, **kwargs)
969 def _capture_args(self, ctx: click.Context, args: list[str]) -> None:
970 """Capture the command line options and arguments.
972 See details about what is captured and the order in which it is stored
973 in the documentation of `MWCtxObj`.
975 Parameters
976 ----------
977 ctx : `click.Context`
978 The current Context.
979 args : `list` [`str`]
980 The list of arguments from the command line, split at spaces but
981 not at separators (like "=").
982 """
983 parser = self.make_parser(ctx)
984 opts, _, param_order = parser.parse_args(args=list(args))
985 # `param_order` is a list of click.Option and click.Argument, there is
986 # one item for each time the Option or Argument was used on the
987 # command line. Options will precede Arguments, within each sublist
988 # they are in the order they were used on the command line. Note that
989 # click.Option and click.Argument do not contain the value from the
990 # command line; values are in `opts`.
991 #
992 # `opts` is a dict where the key is the argument name to the
993 # click.Command function, this name matches the `click.Option.name` or
994 # `click.Argument.name`. For Options, an item will only be present if
995 # the Option was used on the command line. For Arguments, an item will
996 # always be present and if no value was provided on the command line
997 # the value will be `None`. If the option accepts multiple values, the
998 # value in `opts` is a tuple, otherwise it is a single item.
999 next_idx: Counter = Counter()
1000 captured_args = []
1001 for param in param_order:
1002 if isinstance(param, click.Option):
1003 param_name = cast(str, param.name)
1004 if param.multiple:
1005 val = opts[param_name][next_idx[param_name]]
1006 next_idx[param_name] += 1
1007 else:
1008 val = opts[param_name]
1009 if param.is_flag:
1010 # Bool options store their True flags in opts and their
1011 # False flags in secondary_opts.
1012 if val:
1013 flag = max(param.opts, key=len)
1014 else:
1015 flag = max(param.secondary_opts, key=len)
1016 captured_args.append(flag)
1017 else:
1018 captured_args.append(max(param.opts, key=len))
1019 captured_args.append(val)
1020 elif isinstance(param, click.Argument):
1021 param_name = cast(str, param.name)
1022 opt = opts[param_name]
1023 if opt is not None and opt != _CLICK_UNSET_SENTINEL:
1024 captured_args.append(opt)
1025 else:
1026 raise AssertionError("All parameters should be an Option or an Argument")
1027 MWCtxObj.getFrom(ctx).args = captured_args
1029 def parse_args(self, ctx: click.Context, args: Any) -> list[str]:
1030 """Given a context and a list of arguments this creates the parser and
1031 parses the arguments, then modifies the context as necessary. This is
1032 automatically invoked by make_context().
1034 This function overrides `click.Command.parse_args`.
1036 The call to `_capture_args` in this override stores the arguments
1037 (option names, option value, and argument values) that were used by the
1038 caller on the command line in the context object. These stored
1039 arugments can be used by the command function, e.g. to process options
1040 in the order they appeared on the command line (pipetask uses this
1041 feature to create pipeline actions in an order from different options).
1043 Parameters
1044 ----------
1045 ctx : `click.core.Context`
1046 The current Context.
1047 args : `list` [`str`]
1048 The list of arguments from the command line, split at spaces but
1049 not at separators (like "=").
1050 """
1051 self._capture_args(ctx, args)
1052 return super().parse_args(ctx, args)
1054 @property
1055 def epilog(self) -> str | None:
1056 """Override the epilog attribute to add extra_epilog (if defined by a
1057 subclass) to the end of any epilog provided by a subcommand.
1058 """
1059 ret = self._epilog if self._epilog else ""
1060 if self.extra_epilog:
1061 if ret:
1062 ret += "\n\n"
1063 ret += self.extra_epilog
1064 return ret
1066 @epilog.setter
1067 def epilog(self, val: str | None) -> None:
1068 self._epilog = val
1071class ButlerCommand(MWCommand):
1072 """Command subclass with butler-command specific overrides."""
1074 extra_epilog = "See 'butler --help' for more options."
1077class OptionGroup:
1078 """Base class for an option group decorator. Requires the option group
1079 subclass to have a property called `decorator`.
1080 """
1082 decorators: list[Any]
1084 def __call__(self, f: Any) -> Any:
1085 for decorator in reversed(self.decorators):
1086 f = decorator(f)
1087 return f
1090class MWCtxObj:
1091 """Helper object for managing the `click.Context.obj` parameter, allows
1092 obj data to be managed in a consistent way.
1094 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
1095 initialize the obj if needed and return a new or existing `MWCtxObj`.
1097 The `args` attribute contains a list of options, option values, and
1098 argument values that is similar to the list of arguments and options that
1099 were passed in on the command line, with differences noted below:
1101 * Option namess and option values are first in the list, and argument
1102 values come last. The order of options and option values is preserved
1103 within the options. The order of argument values is preserved.
1105 * The longest option name is used for the option in the `args` list, e.g.
1106 if an option accepts both short and long names "-o / --option" and the
1107 short option name "-o" was used on the command line, the longer name will
1108 be the one that appears in `args`.
1110 * A long option name (which begins with two dashes "--") and its value may
1111 be separated by an equal sign; the name and value are split at the equal
1112 sign and it is removed. In `args`, the option is in one list item, and
1113 the option value (without the equal sign) is in the next list item. e.g.
1114 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in
1115 `args`.
1117 * A short option name, (which begins with one dash "-") and its value are
1118 split immediately after the short option name, and if there is
1119 whitespace between the short option name and its value it is removed.
1120 Everything after the short option name (excluding whitespace) is included
1121 in the value. If the `Option` has a long name, the long name will be used
1122 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo"
1123 become `["--option", "foo"]`, and (note!) "-o=foo" will become
1124 `["--option", "=foo"]` (because everything after the short option name,
1125 except whitespace, is used for the value (as is standard with unix
1126 command line tools).
1128 Attributes
1129 ----------
1130 args : `list` [`str`]
1131 A list of options, option values, and arguments simialr to those that
1132 were passed in on the command line. See comments about captured options
1133 & arguments above.
1134 """
1136 def __init__(self) -> None:
1137 self.args = None
1139 @staticmethod
1140 def getFrom(ctx: click.Context) -> Any:
1141 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return
1142 the new or already existing `MWCtxObj`.
1144 Parameters
1145 ----------
1146 ctx : `click.Context`
1147 Context provided by Click.
1148 """
1149 if ctx.obj is not None:
1150 return ctx.obj
1151 ctx.obj = MWCtxObj()
1152 return ctx.obj
1155def yaml_presets(ctx: click.Context, param: str, value: Any) -> None:
1156 """Read additional values from the supplied YAML file.
1158 Parameters
1159 ----------
1160 ctx : `click.context`
1161 The context for the click operation. Used to extract the subcommand
1162 name and translate option & argument names.
1163 param : `str`
1164 The parameter name.
1165 value : `object`
1166 The value of the parameter.
1167 """
1169 def _name_for_option(ctx: click.Context, option: str) -> str:
1170 """Use a CLI option name to find the name of the argument to the
1171 command function.
1173 Parameters
1174 ----------
1175 ctx : `click.Context`
1176 The context for the click operation.
1177 option : `str`
1178 The option/argument name from the yaml file.
1180 Returns
1181 -------
1182 name : str
1183 The name of the argument to use when calling the click.command
1184 function, as it should appear in the `ctx.default_map`.
1186 Raises
1187 ------
1188 RuntimeError
1189 Raised if the option name from the yaml file does not exist in the
1190 command parameters. This catches misspellings and incorrect useage
1191 in the yaml file.
1192 """
1193 for param in ctx.command.params:
1194 # Remove leading dashes: they are not used for option names in the
1195 # yaml file.
1196 if option in [opt.lstrip("-") for opt in param.opts]:
1197 return cast(str, param.name)
1198 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
1200 ctx.default_map = ctx.default_map or {}
1201 cmd_name = ctx.info_name
1202 assert cmd_name is not None, "command name cannot be None"
1203 if value:
1204 try:
1205 overrides = _read_yaml_presets(value, cmd_name)
1206 options = list(overrides.keys())
1207 for option in options:
1208 name = _name_for_option(ctx, option)
1209 if name == option:
1210 continue
1211 overrides[name] = overrides.pop(option)
1212 except Exception as e:
1213 raise click.BadOptionUsage(
1214 option_name=param,
1215 message=f"Error reading overrides file: {e}",
1216 ctx=ctx,
1217 ) from None
1218 # Override the defaults for this subcommand
1219 ctx.default_map.update(overrides)
1220 return
1223def _read_yaml_presets(file_uri: str, cmd_name: str) -> dict[str, Any]:
1224 """Read file command line overrides from YAML config file.
1226 Parameters
1227 ----------
1228 file_uri : `str`
1229 URI of override YAML file containing the command line overrides.
1230 They should be grouped by command name.
1231 cmd_name : `str`
1232 The subcommand name that is being modified.
1234 Returns
1235 -------
1236 overrides : `dict` of [`str`, Any]
1237 The relevant command line options read from the override file.
1238 """
1239 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1240 config = Config(file_uri)
1241 return config[cmd_name]
1244def sortAstropyTable(table: Table, dimensions: list[Dimension], sort_first: list[str] | None = None) -> Table:
1245 """Sort an astropy table.
1247 Prioritization is given to columns in this order:
1249 1. the provided named columns
1250 2. spatial and temporal columns
1251 3. the rest of the columns.
1253 The table is sorted in-place, and is also returned for convenience.
1255 Parameters
1256 ----------
1257 table : `astropy.table.Table`
1258 The table to sort.
1259 dimensions : `list` [``Dimension``]
1260 The dimensions of the dataIds in the table (the dimensions should be
1261 the same for all the dataIds). Used to determine if the column is
1262 spatial, temporal, or neither.
1263 sort_first : `list` [`str`]
1264 The names of columns that should be sorted first, before spatial and
1265 temporal columns.
1267 Returns
1268 -------
1269 `astropy.table.Table`
1270 For convenience, the table that has been sorted.
1271 """
1272 # For sorting we want to ignore the id
1273 # We also want to move temporal or spatial dimensions earlier
1274 sort_first = sort_first or []
1275 sort_early: list[str] = []
1276 sort_late: list[str] = []
1277 for dim in dimensions:
1278 if dim.spatial or dim.temporal:
1279 sort_early.extend(dim.required.names)
1280 else:
1281 sort_late.append(str(dim))
1282 sort_keys = sort_first + sort_early + sort_late
1283 # The required names above means that we have the possibility of
1284 # repeats of sort keys. Now have to remove them
1285 # (order is retained by dict creation).
1286 sort_keys = list(dict.fromkeys(sort_keys).keys())
1288 table.sort(sort_keys)
1289 return table
1292def catch_and_exit(func: Callable) -> Callable:
1293 """Catch all exceptions, prints an exception traceback
1294 and signals click to exit.
1296 Use as decorator.
1298 Parameters
1299 ----------
1300 func : `collections.abc.Callable`
1301 The function to be decorated.
1303 Returns
1304 -------
1305 `collections.abc.Callable`
1306 The decorated function.
1307 """
1309 @wraps(func)
1310 def inner(*args: Any, **kwargs: Any) -> None:
1311 try:
1312 func(*args, **kwargs)
1313 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1314 # this is handled by click itself
1315 raise
1316 except Exception:
1317 exc_type, exc_value, exc_tb = sys.exc_info()
1318 assert exc_type is not None
1319 assert exc_value is not None
1320 assert exc_tb is not None
1321 exit_hdl = ClickExitFailedNicely(exc_type, exc_value, exc_tb)
1322 exit_hdl.exit_click()
1324 return inner