Coverage for python/lsst/daf/butler/cli/utils.py: 32%
343 statements
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
« prev ^ index » next coverage.py v6.5.0, created at 2022-11-15 01:59 -0800
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
23__all__ = (
24 "astropyTablesToStr",
25 "printAstropyTables",
26 "textTypeStr",
27 "LogCliRunner",
28 "clickResultMsg",
29 "command_test_env",
30 "addArgumentHelp",
31 "split_commas",
32 "split_kv",
33 "to_upper",
34 "unwrap",
35 "option_section",
36 "MWPath",
37 "MWOption",
38 "MWArgument",
39 "OptionSection",
40 "MWOptionDecorator",
41 "MWArgumentDecorator",
42 "MWCommand",
43 "ButlerCommand",
44 "OptionGroup",
45 "MWCtxObj",
46 "yaml_presets",
47 "sortAstropyTable",
48 "catch_and_exit",
49)
52import itertools
53import logging
54import os
55import re
56import sys
57import textwrap
58import traceback
59import uuid
60import warnings
61from collections import Counter
62from contextlib import contextmanager
63from functools import partial, wraps
64from unittest.mock import patch
66import click
67import click.exceptions
68import click.testing
69import yaml
70from lsst.utils.iteration import ensure_iterable
72from ..core.config import Config
73from .cliLog import CliLog
75log = logging.getLogger(__name__)
77# This is used as the metavar argument to Options that accept multiple string
78# inputs, which may be comma-separarated. For example:
79# --my-opt foo,bar --my-opt baz.
80# Other arguments to the Option should include multiple=true and
81# callback=split_kv.
82typeStrAcceptsMultiple = "TEXT ..."
83typeStrAcceptsSingle = "TEXT"
85# The standard help string for the --where option when it takes a WHERE clause.
86where_help = (
87 "A string expression similar to a SQL WHERE clause. May involve any column of a "
88 "dimension table or a dimension name as a shortcut for the primary key column of a "
89 "dimension table."
90)
93def astropyTablesToStr(tables):
94 """Render astropy tables to string as they are displayed in the CLI.
96 Output formatting matches ``printAstropyTables``.
97 """
98 ret = ""
99 for table in tables:
100 ret += "\n"
101 table.pformat_all()
102 ret += "\n"
103 return ret
106def printAstropyTables(tables):
107 """Print astropy tables to be displayed in the CLI.
109 Output formatting matches ``astropyTablesToStr``.
110 """
111 for table in tables:
112 print("")
113 table.pprint_all()
114 print("")
117def textTypeStr(multiple):
118 """Get the text type string for CLI help documentation.
120 Parameters
121 ----------
122 multiple : `bool`
123 True if multiple text values are allowed, False if only one value is
124 allowed.
126 Returns
127 -------
128 textTypeStr : `str`
129 The type string to use.
130 """
131 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
134class LogCliRunner(click.testing.CliRunner):
135 """A test runner to use when the logging system will be initialized by code
136 under test, calls CliLog.resetLog(), which undoes any logging setup that
137 was done with the CliLog interface.
139 lsst.log modules can not be set back to an uninitialized state (python
140 logging modules can be set back to NOTSET), instead they are set to
141 `CliLog.defaultLsstLogLevel`."""
143 def invoke(self, *args, **kwargs):
144 result = super().invoke(*args, **kwargs)
145 CliLog.resetLog()
146 return result
149def clickResultMsg(result):
150 """Get a standard assert message from a click result
152 Parameters
153 ----------
154 result : click.Result
155 The result object returned from click.testing.CliRunner.invoke
157 Returns
158 -------
159 msg : `str`
160 The message string.
161 """
162 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
163 if result.exception:
164 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
165 return msg
168@contextmanager
169def command_test_env(runner, commandModule, commandName):
170 """A context manager that creates (and then cleans up) an environment that
171 provides a CLI plugin command with the given name.
173 Parameters
174 ----------
175 runner : click.testing.CliRunner
176 The test runner to use to create the isolated filesystem.
177 commandModule : `str`
178 The importable module that the command can be imported from.
179 commandName : `str`
180 The name of the command being published to import.
181 """
182 with runner.isolated_filesystem():
183 with open("resources.yaml", "w") as f:
184 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
185 # Add a colon to the end of the path on the next line, this tests the
186 # case where the lookup in LoaderCLI._getPluginList generates an empty
187 # string in one of the list entries and verifies that the empty string
188 # is properly stripped out.
189 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
190 yield
193def addArgumentHelp(doc, helpText):
194 """Add a Click argument's help message to a function's documentation.
196 This is needed because click presents arguments in the order the argument
197 decorators are applied to a function, top down. But, the evaluation of the
198 decorators happens bottom up, so if arguments just append their help to the
199 function's docstring, the argument descriptions appear in reverse order
200 from the order they are applied in.
202 Parameters
203 ----------
204 doc : `str`
205 The function's docstring.
206 helpText : `str`
207 The argument's help string to be inserted into the function's
208 docstring.
210 Returns
211 -------
212 doc : `str`
213 Updated function documentation.
214 """
215 if doc is None: 215 ↛ 216line 215 didn't jump to line 216, because the condition on line 215 was never true
216 doc = helpText
217 else:
218 # See click documentation for details:
219 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
220 # In short, text for the click command help can be truncated by putting
221 # "\f" in the docstring, everything after it should be removed
222 if "\f" in doc: 222 ↛ 223line 222 didn't jump to line 223, because the condition on line 222 was never true
223 doc = doc.split("\f")[0]
225 doclines = doc.splitlines()
226 # The function's docstring may span multiple lines, so combine the
227 # docstring from all the first lines until a blank line is encountered.
228 # (Lines after the first blank line will be argument help.)
229 while len(doclines) > 1 and doclines[1]:
230 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
231 # Add standard indent to help text for proper alignment with command
232 # function documentation:
233 helpText = " " + helpText
234 doclines.insert(1, helpText)
235 doclines.insert(1, "\n")
236 doc = "\n".join(doclines)
237 return doc
240def split_commas(context, param, values):
241 """Process a tuple of values, where each value may contain comma-separated
242 values, and return a single list of all the passed-in values.
244 This function can be passed to the 'callback' argument of a click.option to
245 allow it to process comma-separated values (e.g. "--my-opt a,b,c"). If
246 the comma is inside ``[]`` there will be no splitting.
248 Parameters
249 ----------
250 context : `click.Context` or `None`
251 The current execution context. Unused, but Click always passes it to
252 callbacks.
253 param : `click.core.Option` or `None`
254 The parameter being handled. Unused, but Click always passes it to
255 callbacks.
256 values : iterable of `str` or `str`
257 All the values passed for this option. Strings may contain commas,
258 which will be treated as delimiters for separate values unless they
259 are within ``[]``.
261 Returns
262 -------
263 results : `tuple` [`str`]
264 The passed in values separated by commas where appropriate and
265 combined into a single tuple.
266 """
267 if values is None:
268 return values
269 valueList = []
270 for value in ensure_iterable(values):
271 # If we have [, or ,] we do the slow split. If square brackets
272 # are not matching then that is likely a typo that should result
273 # in a warning.
274 opens = "["
275 closes = "]"
276 if re.search(rf"\{opens}.*,|,.*\{closes}", value):
277 in_parens = False
278 current = ""
279 for c in value:
280 if c == opens:
281 if in_parens:
282 warnings.warn(
283 f"Found second opening {opens} without corresponding closing {closes}"
284 f" in {value!r}",
285 stacklevel=2,
286 )
287 in_parens = True
288 elif c == closes:
289 if not in_parens:
290 warnings.warn(
291 f"Found second closing {closes} without corresponding open {opens} in {value!r}",
292 stacklevel=2,
293 )
294 in_parens = False
295 elif c == ",":
296 if not in_parens:
297 # Split on this comma.
298 valueList.append(current)
299 current = ""
300 continue
301 current += c
302 if in_parens:
303 warnings.warn(
304 f"Found opening {opens} that was never closed in {value!r}",
305 stacklevel=2,
306 )
307 if current:
308 valueList.append(current)
309 else:
310 # Use efficient split since no parens.
311 valueList.extend(value.split(","))
312 return tuple(valueList)
315def split_kv(
316 context,
317 param,
318 values,
319 choice=None,
320 multiple=True,
321 normalize=False,
322 separator="=",
323 unseparated_okay=False,
324 return_type=dict,
325 default_key="",
326 reverse_kv=False,
327 add_to_default=False,
328):
329 """Process a tuple of values that are key-value pairs separated by a given
330 separator. Multiple pairs may be comma separated. Return a dictionary of
331 all the passed-in values.
333 This function can be passed to the 'callback' argument of a click.option to
334 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
336 Parameters
337 ----------
338 context : `click.Context` or `None`
339 The current execution context. Unused, but Click always passes it to
340 callbacks.
341 param : `click.core.Option` or `None`
342 The parameter being handled. Unused, but Click always passes it to
343 callbacks.
344 values : [`str`]
345 All the values passed for this option. Strings may contain commas,
346 which will be treated as delimiters for separate values.
347 choice : `click.Choice`, optional
348 If provided, verify each value is a valid choice using the provided
349 `click.Choice` instance. If None, no verification will be done. By
350 default None
351 multiple : `bool`, optional
352 If true, the value may contain multiple comma-separated values. By
353 default True.
354 normalize : `bool`, optional
355 If True and `choice.case_sensitive == False`, normalize the string the
356 user provided to match the choice's case. By default False.
357 separator : str, optional
358 The character that separates key-value pairs. May not be a comma or an
359 empty space (for space separators use Click's default implementation
360 for tuples; `type=(str, str)`). By default "=".
361 unseparated_okay : `bool`, optional
362 If True, allow values that do not have a separator. They will be
363 returned in the values dict as a tuple of values in the key '', that
364 is: `values[''] = (unseparated_values, )`. By default False.
365 return_type : `type`, must be `dict` or `tuple`
366 The type of the value that should be returned.
367 If `dict` then the returned object will be a dict, for each item in
368 values, the value to the left of the separator will be the key and the
369 value to the right of the separator will be the value.
370 If `tuple` then the returned object will be a tuple. Each item in the
371 tuple will be 2-item tuple, the first item will be the value to the
372 left of the separator and the second item will be the value to the
373 right. By default `dict`.
374 default_key : `Any`
375 The key to use if a value is passed that is not a key-value pair.
376 (Passing values that are not key-value pairs requires
377 ``unseparated_okay`` to be `True`.)
378 reverse_kv : bool
379 If true then for each item in values, the value to the left of the
380 separator is treated as the value and the value to the right of the
381 separator is treated as the key. By default False.
382 add_to_default : `bool`, optional
383 If True, then passed-in values will not overwrite the default value
384 unless the ``return_type`` is `dict` and passed-in value(s) have the
385 same key(s) as the default value.
387 Returns
388 -------
389 values : `dict` [`str`, `str`]
390 The passed-in values in dict form.
392 Raises
393 ------
394 `click.ClickException`
395 Raised if the separator is not found in an entry, or if duplicate keys
396 are encountered.
397 """
399 def norm(val):
400 """If `normalize` is True and `choice` is not `None`, find the value
401 in the available choices and return the value as spelled in the
402 choices.
404 Assumes that val exists in choices; `split_kv` uses the `choice`
405 instance to verify val is a valid choice.
406 """
407 if normalize and choice is not None:
408 v = val.casefold()
409 for opt in choice.choices:
410 if opt.casefold() == v:
411 return opt
412 return val
414 class RetDict:
415 def __init__(self):
416 self.ret = {}
418 def add(self, key, val):
419 if reverse_kv:
420 key, val = val, key
421 self.ret[key] = val
423 def get(self):
424 return self.ret
426 class RetTuple:
427 def __init__(self):
428 self.ret = []
430 def add(self, key, val):
431 if reverse_kv:
432 key, val = val, key
433 self.ret.append((key, val))
435 def get(self):
436 return tuple(self.ret)
438 if separator in (",", " "):
439 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
440 vals = values # preserve the original argument for error reporting below.
442 if add_to_default:
443 default = param.get_default(context)
444 if default:
445 vals = itertools.chain(default, vals)
447 if return_type is dict:
448 ret = RetDict()
449 elif return_type is tuple:
450 ret = RetTuple()
451 else:
452 raise click.ClickException(
453 message=f"Internal error: invalid return type '{return_type}' for split_kv."
454 )
455 if multiple:
456 vals = split_commas(context, param, vals)
457 for val in ensure_iterable(vals):
458 if unseparated_okay and separator not in val:
459 if choice is not None:
460 choice(val) # will raise if val is an invalid choice
461 ret.add(default_key, norm(val))
462 else:
463 try:
464 k, v = val.split(separator)
465 if choice is not None:
466 choice(v) # will raise if val is an invalid choice
467 except ValueError as e:
468 raise click.ClickException(
469 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
470 f"with multiple values {'allowed' if multiple else 'not allowed'}: {e}"
471 )
472 ret.add(k, norm(v))
473 return ret.get()
476def to_upper(context, param, value):
477 """Convert a value to upper case.
479 Parameters
480 ----------
481 context : click.Context
483 values : string
484 The value to be converted.
486 Returns
487 -------
488 string
489 A copy of the passed-in value, converted to upper case.
490 """
491 return value.upper()
494def unwrap(val):
495 """Remove newlines and leading whitespace from a multi-line string with
496 a consistent indentation level.
498 The first line of the string may be only a newline or may contain text
499 followed by a newline, either is ok. After the first line, each line must
500 begin with a consistant amount of whitespace. So, content of a
501 triple-quoted string may begin immediately after the quotes, or the string
502 may start with a newline. Each line after that must be the same amount of
503 indentation/whitespace followed by text and a newline. The last line may
504 end with a new line but is not required to do so.
506 Parameters
507 ----------
508 val : `str`
509 The string to change.
511 Returns
512 -------
513 strippedString : `str`
514 The string with newlines, indentation, and leading and trailing
515 whitespace removed.
516 """
518 def splitSection(val):
519 if not val.startswith("\n"): 519 ↛ 523line 519 didn't jump to line 523, because the condition on line 519 was never false
520 firstLine, _, val = val.partition("\n")
521 firstLine += " "
522 else:
523 firstLine = ""
524 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
526 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
529class option_section: # noqa: N801
530 """Decorator to add a section label between options in the help text of a
531 command.
533 Parameters
534 ----------
535 sectionText : `str`
536 The text to print in the section identifier.
537 """
539 def __init__(self, sectionText):
540 self.sectionText = "\n" + sectionText
542 def __call__(self, f):
543 # Generate a parameter declaration that will be unique for this
544 # section.
545 return click.option(
546 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
547 )(f)
550class MWPath(click.Path):
551 """Overrides click.Path to implement file-does-not-exist checking.
553 Changes the definition of ``exists` so that `True` indicates the location
554 (file or directory) must exist, `False` indicates the location must *not*
555 exist, and `None` indicates that the file may exist or not. The standard
556 definition for the `click.Path` ``exists`` parameter is that for `True` a
557 location must exist, but `False` means it is not required to exist (not
558 that it is required to not exist).
560 Parameters
561 ----------
562 exists : `True`, `False`, or `None`
563 If `True`, the location (file or directory) indicated by the caller
564 must exist. If `False` the location must not exist. If `None`, the
565 location may exist or not.
567 For other parameters see `click.Path`.
568 """
570 def __init__(
571 self,
572 exists=None,
573 file_okay=True,
574 dir_okay=True,
575 writable=False,
576 readable=True,
577 resolve_path=False,
578 allow_dash=False,
579 path_type=None,
580 ):
581 self.mustNotExist = exists is False
582 if exists is None: 582 ↛ 584line 582 didn't jump to line 584, because the condition on line 582 was never false
583 exists = False
584 super().__init__(
585 exists=exists,
586 file_okay=file_okay,
587 dir_okay=dir_okay,
588 writable=writable,
589 readable=readable,
590 resolve_path=resolve_path,
591 allow_dash=allow_dash,
592 path_type=path_type,
593 )
595 def convert(self, value, param, ctx):
596 """Called by click.ParamType to "convert values through types".
597 `click.Path` uses this step to verify Path conditions."""
598 if self.mustNotExist and os.path.exists(value):
599 self.fail(f'Path "{value}" should not exist.')
600 return super().convert(value, param, ctx)
603class MWOption(click.Option):
604 """Overrides click.Option with desired behaviors."""
606 def make_metavar(self):
607 """Overrides `click.Option.make_metavar`. Makes the metavar for the
608 help menu. Adds a space and an elipsis after the metavar name if
609 the option accepts multiple inputs, otherwise defers to the base
610 implementation.
612 By default click does not add an elipsis when multiple is True and
613 nargs is 1. And when nargs does not equal 1 click adds an elipsis
614 without a space between the metavar and the elipsis, but we prefer a
615 space between.
617 Does not get called for some option types (e.g. flag) so metavar
618 transformation that must apply to all types should be applied in
619 get_help_record.
620 """
621 metavar = super().make_metavar()
622 if self.multiple and self.nargs == 1:
623 metavar += " ..."
624 elif self.nargs != 1:
625 metavar = f"{metavar[:-3]} ..."
626 return metavar
629class MWArgument(click.Argument):
630 """Overrides click.Argument with desired behaviors."""
632 def make_metavar(self):
633 """Overrides `click.Option.make_metavar`. Makes the metavar for the
634 help menu. Always adds a space and an elipsis (' ...') after the
635 metavar name if the option accepts multiple inputs.
637 By default click adds an elipsis without a space between the metavar
638 and the elipsis, but we prefer a space between.
640 Returns
641 -------
642 metavar : `str`
643 The metavar value.
644 """
645 metavar = super().make_metavar()
646 if self.nargs != 1:
647 metavar = f"{metavar[:-3]} ..."
648 return metavar
651class OptionSection(MWOption):
652 """Implements an Option that prints a section label in the help text and
653 does not pass any value to the command function.
655 This class does a bit of hackery to add a section label to a click command
656 help output: first, `expose_value` is set to `False` so that no value is
657 passed to the command function. Second, this class overrides
658 `click.Option.get_help_record` to return the section label string without
659 any prefix so that it stands out as a section label.
661 This class overrides the hidden attribute because our documentation build
662 tool, sphinx-click, implements its own `get_help_record` function which
663 builds the record from other option values (e.g. `name`, `opts`), which
664 breaks the hack we use to make `get_help_record` only return the
665 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
666 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
667 entering its `_get_help_record` function. So, making the hidden property
668 return True hides this option from sphinx-click, while allowing the section
669 text to be returned by our `get_help_record` method when using Click.
671 The intention for this implementation is to do minimally invasive overrides
672 of the click classes so as to be robust and easy to fix if the click
673 internals change.
675 Parameters
676 ----------
677 sectionName : `str`
678 The parameter declaration for this option. It is not shown to the user,
679 it must be unique within the command. If using the `section` decorator
680 to add a section to a command's options, the section name is
681 auto-generated.
682 sectionText : `str`
683 The text to print in the section identifier.
684 """
686 @property
687 def hidden(self):
688 return True
690 @hidden.setter
691 def hidden(self, val):
692 pass
694 def __init__(self, sectionName, sectionText):
695 super().__init__(sectionName, expose_value=False)
696 self.sectionText = sectionText
698 def get_help_record(self, ctx):
699 return (self.sectionText, "")
702class MWOptionDecorator:
703 """Wraps the click.option decorator to enable shared options to be declared
704 and allows inspection of the shared option.
705 """
707 def __init__(self, *param_decls, **kwargs):
708 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs)
709 opt = click.Option(param_decls, **kwargs)
710 self._name = opt.name
711 self._opts = opt.opts
713 def name(self):
714 """Get the name that will be passed to the command function for this
715 option."""
716 return self._name
718 def opts(self):
719 """Get the flags that will be used for this option on the command
720 line."""
721 return self._opts
723 @property
724 def help(self):
725 """Get the help text for this option. Returns an empty string if no
726 help was defined."""
727 return self.partialOpt.keywords.get("help", "")
729 def __call__(self, *args, **kwargs):
730 return self.partialOpt(*args, **kwargs)
733class MWArgumentDecorator:
734 """Wraps the click.argument decorator to enable shared arguments to be
735 declared."""
737 def __init__(self, *param_decls, **kwargs):
738 self._helpText = kwargs.pop("help", None)
739 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
741 def __call__(self, *args, help=None, **kwargs):
742 def decorator(f):
743 if help is not None:
744 self._helpText = help
745 if self._helpText:
746 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
747 return self.partialArg(*args, **kwargs)(f)
749 return decorator
752class MWCommand(click.Command):
753 """Command subclass that stores a copy of the args list for use by the
754 command."""
756 extra_epilog = None
758 def __init__(self, *args, **kwargs):
759 # wrap callback method with catch_and_exit decorator
760 callback = kwargs.get("callback")
761 if callback is not None: 761 ↛ 764line 761 didn't jump to line 764, because the condition on line 761 was never false
762 kwargs = kwargs.copy()
763 kwargs["callback"] = catch_and_exit(callback)
764 super().__init__(*args, **kwargs)
766 def _capture_args(self, ctx, args):
767 """Capture the command line options and arguments.
769 See details about what is captured and the order in which it is stored
770 in the documentation of `MWCtxObj`.
772 Parameters
773 ----------
774 ctx : `click.Context`
775 The current Context.
776 args : `list` [`str`]
777 The list of arguments from the command line, split at spaces but
778 not at separators (like "=").
779 """
780 parser = self.make_parser(ctx)
781 opts, _, param_order = parser.parse_args(args=list(args))
782 # `param_order` is a list of click.Option and click.Argument, there is
783 # one item for each time the Option or Argument was used on the
784 # command line. Options will precede Arguments, within each sublist
785 # they are in the order they were used on the command line. Note that
786 # click.Option and click.Argument do not contain the value from the
787 # command line; values are in `opts`.
788 #
789 # `opts` is a dict where the key is the argument name to the
790 # click.Command function, this name matches the `click.Option.name` or
791 # `click.Argument.name`. For Options, an item will only be present if
792 # the Option was used on the command line. For Arguments, an item will
793 # always be present and if no value was provided on the command line
794 # the value will be `None`. If the option accepts multiple values, the
795 # value in `opts` is a tuple, otherwise it is a single item.
796 next_idx = Counter()
797 captured_args = []
798 for param in param_order:
799 if isinstance(param, click.Option):
800 if param.multiple:
801 val = opts[param.name][next_idx[param.name]]
802 next_idx[param.name] += 1
803 else:
804 val = opts[param.name]
805 if param.is_flag:
806 # Bool options store their True flags in opts and their
807 # False flags in secondary_opts.
808 if val:
809 flag = max(param.opts, key=len)
810 else:
811 flag = max(param.secondary_opts, key=len)
812 captured_args.append(flag)
813 else:
814 captured_args.append(max(param.opts, key=len))
815 captured_args.append(val)
816 elif isinstance(param, click.Argument):
817 if (opt := opts[param.name]) is not None:
818 captured_args.append(opt)
819 else:
820 assert False # All parameters should be an Option or an Argument.
821 MWCtxObj.getFrom(ctx).args = captured_args
823 def parse_args(self, ctx, args):
824 """Given a context and a list of arguments this creates the parser and
825 parses the arguments, then modifies the context as necessary. This is
826 automatically invoked by make_context().
828 This function overrides `click.Command.parse_args`.
830 The call to `_capture_args` in this override stores the arguments
831 (option names, option value, and argument values) that were used by the
832 caller on the command line in the context object. These stored
833 arugments can be used by the command function, e.g. to process options
834 in the order they appeared on the command line (pipetask uses this
835 feature to create pipeline actions in an order from different options).
837 Parameters
838 ----------
839 ctx : `click.core.Context`
840 The current Context.ß
841 args : `list` [`str`]
842 The list of arguments from the command line, split at spaces but
843 not at separators (like "=").
844 """
845 self._capture_args(ctx, args)
846 super().parse_args(ctx, args)
848 @property
849 def epilog(self):
850 """Override the epilog attribute to add extra_epilog (if defined by a
851 subclass) to the end of any epilog provided by a subcommand.
852 """
853 ret = self._epilog if self._epilog else ""
854 if self.extra_epilog:
855 if ret:
856 ret += "\n\n"
857 ret += self.extra_epilog
858 return ret
860 @epilog.setter
861 def epilog(self, val):
862 self._epilog = val
865class ButlerCommand(MWCommand):
866 """Command subclass with butler-command specific overrides."""
868 extra_epilog = "See 'butler --help' for more options."
871class OptionGroup:
872 """Base class for an option group decorator. Requires the option group
873 subclass to have a property called `decorator`."""
875 def __call__(self, f):
876 for decorator in reversed(self.decorators):
877 f = decorator(f)
878 return f
881class MWCtxObj:
882 """Helper object for managing the `click.Context.obj` parameter, allows
883 obj data to be managed in a consistent way.
885 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
886 initialize the obj if needed and return a new or existing `MWCtxObj`.
888 The `args` attribute contains a list of options, option values, and
889 argument values that is similar to the list of arguments and options that
890 were passed in on the command line, with differences noted below:
892 * Option namess and option values are first in the list, and argument
893 values come last. The order of options and option values is preserved
894 within the options. The order of argument values is preserved.
896 * The longest option name is used for the option in the `args` list, e.g.
897 if an option accepts both short and long names "-o / --option" and the
898 short option name "-o" was used on the command line, the longer name will
899 be the one that appears in `args`.
901 * A long option name (which begins with two dashes "--") and its value may
902 be separated by an equal sign; the name and value are split at the equal
903 sign and it is removed. In `args`, the option is in one list item, and
904 the option value (without the equal sign) is in the next list item. e.g.
905 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in
906 `args`.
908 * A short option name, (which begins with one dash "-") and its value are
909 split immediately after the short option name, and if there is
910 whitespace between the short option name and its value it is removed.
911 Everything after the short option name (excluding whitespace) is included
912 in the value. If the `Option` has a long name, the long name will be used
913 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo"
914 become `["--option", "foo"]`, and (note!) "-o=foo" will become
915 `["--option", "=foo"]` (because everything after the short option name,
916 except whitespace, is used for the value (as is standard with unix
917 command line tools).
919 Attributes
920 ----------
921 args : `list` [`str`]
922 A list of options, option values, and arguments simialr to those that
923 were passed in on the command line. See comments about captured options
924 & arguments above.
925 """
927 def __init__(self):
929 self.args = None
931 @staticmethod
932 def getFrom(ctx):
933 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return
934 the new or already existing `MWCtxObj`."""
935 if ctx.obj is not None:
936 return ctx.obj
937 ctx.obj = MWCtxObj()
938 return ctx.obj
941def yaml_presets(ctx, param, value):
942 """Click callback that reads additional values from the supplied
943 YAML file.
945 Parameters
946 ----------
947 ctx : `click.context`
948 The context for the click operation. Used to extract the subcommand
949 name and translate option & argument names.
950 param : `str`
951 The parameter name.
952 value : `object`
953 The value of the parameter.
954 """
956 def _name_for_option(ctx: click.Context, option: str) -> str:
957 """Use a CLI option name to find the name of the argument to the
958 command function.
960 Parameters
961 ----------
962 ctx : `click.Context`
963 The context for the click operation.
964 option : `str`
965 The option/argument name from the yaml file.
967 Returns
968 -------
969 name : str
970 The name of the argument to use when calling the click.command
971 function, as it should appear in the `ctx.default_map`.
973 Raises
974 ------
975 RuntimeError
976 Raised if the option name from the yaml file does not exist in the
977 command parameters. This catches misspellings and incorrect useage
978 in the yaml file.
979 """
980 for param in ctx.command.params:
981 # Remove leading dashes: they are not used for option names in the
982 # yaml file.
983 if option in [opt.lstrip("-") for opt in param.opts]:
984 return param.name
985 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
987 ctx.default_map = ctx.default_map or {}
988 cmd_name = ctx.info_name
989 if value:
990 try:
991 overrides = _read_yaml_presets(value, cmd_name)
992 options = list(overrides.keys())
993 for option in options:
994 name = _name_for_option(ctx, option)
995 if name == option:
996 continue
997 overrides[name] = overrides.pop(option)
998 except Exception as e:
999 raise click.BadOptionUsage(
1000 option_name=param.name,
1001 message=f"Error reading overrides file: {e}",
1002 ctx=ctx,
1003 )
1004 # Override the defaults for this subcommand
1005 ctx.default_map.update(overrides)
1006 return
1009def _read_yaml_presets(file_uri, cmd_name):
1010 """Read file command line overrides from YAML config file.
1012 Parameters
1013 ----------
1014 file_uri : `str`
1015 URI of override YAML file containing the command line overrides.
1016 They should be grouped by command name.
1017 cmd_name : `str`
1018 The subcommand name that is being modified.
1020 Returns
1021 -------
1022 overrides : `dict` of [`str`, Any]
1023 The relevant command line options read from the override file.
1024 """
1025 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
1026 config = Config(file_uri)
1027 return config[cmd_name]
1030def sortAstropyTable(table, dimensions, sort_first=None):
1031 """Sort an astropy table, with prioritization given to columns in this
1032 order:
1033 1. the provided named columns
1034 2. spatial and temporal columns
1035 3. the rest of the columns
1037 The table is sorted in-place, and is also returned for convenience.
1039 Parameters
1040 ----------
1041 table : `astropy.table.Table`
1042 The table to sort
1043 dimensions : `list` [``Dimension``]
1044 The dimensions of the dataIds in the table (the dimensions should be
1045 the same for all the dataIds). Used to determine if the column is
1046 spatial, temporal, or neither.
1047 sort_first : `list` [`str`]
1048 The names of columns that should be sorted first, before spatial and
1049 temporal columns.
1051 Returns
1052 -------
1053 `astropy.table.Table`
1054 For convenience, the table that has been sorted.
1055 """
1056 # For sorting we want to ignore the id
1057 # We also want to move temporal or spatial dimensions earlier
1058 sort_first = sort_first or []
1059 sort_early = []
1060 sort_late = []
1061 for dim in dimensions:
1062 if dim.spatial or dim.temporal:
1063 sort_early.extend(dim.required.names)
1064 else:
1065 sort_late.append(str(dim))
1066 sort_keys = sort_first + sort_early + sort_late
1067 # The required names above means that we have the possibility of
1068 # repeats of sort keys. Now have to remove them
1069 # (order is retained by dict creation).
1070 sort_keys = list(dict.fromkeys(sort_keys).keys())
1072 table.sort(sort_keys)
1073 return table
1076def catch_and_exit(func):
1077 """Decorator which catches all exceptions, prints an exception traceback
1078 and signals click to exit.
1079 """
1081 @wraps(func)
1082 def inner(*args, **kwargs):
1083 try:
1084 func(*args, **kwargs)
1085 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1086 # this is handled by click itself
1087 raise
1088 except Exception:
1089 exc_type, exc_value, exc_tb = sys.exc_info()
1090 if exc_tb.tb_next:
1091 # do not show this decorator in traceback
1092 exc_tb = exc_tb.tb_next
1093 log.exception(
1094 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb)
1095 )
1096 # tell click to stop, this never returns.
1097 click.get_current_context().exit(1)
1099 return inner