Coverage for python/lsst/daf/butler/cli/utils.py: 38%
317 statements
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 09:44 +0000
« prev ^ index » next coverage.py v6.4.1, created at 2022-06-23 09:44 +0000
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
23__all__ = (
24 "astropyTablesToStr",
25 "printAstropyTables",
26 "textTypeStr",
27 "LogCliRunner",
28 "clickResultMsg",
29 "command_test_env",
30 "addArgumentHelp",
31 "split_commas",
32 "split_kv",
33 "to_upper",
34 "unwrap",
35 "option_section",
36 "MWPath",
37 "MWOption",
38 "MWArgument",
39 "OptionSection",
40 "MWOptionDecorator",
41 "MWArgumentDecorator",
42 "MWCommand",
43 "ButlerCommand",
44 "OptionGroup",
45 "MWCtxObj",
46 "yaml_presets",
47 "sortAstropyTable",
48 "catch_and_exit",
49)
52import itertools
53import logging
54import os
55import sys
56import textwrap
57import traceback
58import uuid
59from collections import Counter
60from contextlib import contextmanager
61from functools import partial, wraps
62from unittest.mock import patch
64import click
65import click.exceptions
66import click.testing
67import yaml
68from lsst.utils.iteration import ensure_iterable
70from ..core.config import Config
71from .cliLog import CliLog
73log = logging.getLogger(__name__)
75# This is used as the metavar argument to Options that accept multiple string
76# inputs, which may be comma-separarated. For example:
77# --my-opt foo,bar --my-opt baz.
78# Other arguments to the Option should include multiple=true and
79# callback=split_kv.
80typeStrAcceptsMultiple = "TEXT ..."
81typeStrAcceptsSingle = "TEXT"
83# The standard help string for the --where option when it takes a WHERE clause.
84where_help = (
85 "A string expression similar to a SQL WHERE clause. May involve any column of a "
86 "dimension table or a dimension name as a shortcut for the primary key column of a "
87 "dimension table."
88)
91def astropyTablesToStr(tables):
92 """Render astropy tables to string as they are displayed in the CLI.
94 Output formatting matches ``printAstropyTables``.
95 """
96 ret = ""
97 for table in tables:
98 ret += "\n"
99 table.pformat_all()
100 ret += "\n"
101 return ret
104def printAstropyTables(tables):
105 """Print astropy tables to be displayed in the CLI.
107 Output formatting matches ``astropyTablesToStr``.
108 """
109 for table in tables:
110 print("")
111 table.pprint_all()
112 print("")
115def textTypeStr(multiple):
116 """Get the text type string for CLI help documentation.
118 Parameters
119 ----------
120 multiple : `bool`
121 True if multiple text values are allowed, False if only one value is
122 allowed.
124 Returns
125 -------
126 textTypeStr : `str`
127 The type string to use.
128 """
129 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
132class LogCliRunner(click.testing.CliRunner):
133 """A test runner to use when the logging system will be initialized by code
134 under test, calls CliLog.resetLog(), which undoes any logging setup that
135 was done with the CliLog interface.
137 lsst.log modules can not be set back to an uninitialized state (python
138 logging modules can be set back to NOTSET), instead they are set to
139 `CliLog.defaultLsstLogLevel`."""
141 def invoke(self, *args, **kwargs):
142 result = super().invoke(*args, **kwargs)
143 CliLog.resetLog()
144 return result
147def clickResultMsg(result):
148 """Get a standard assert message from a click result
150 Parameters
151 ----------
152 result : click.Result
153 The result object returned from click.testing.CliRunner.invoke
155 Returns
156 -------
157 msg : `str`
158 The message string.
159 """
160 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
161 if result.exception:
162 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
163 return msg
166@contextmanager
167def command_test_env(runner, commandModule, commandName):
168 """A context manager that creates (and then cleans up) an environment that
169 provides a CLI plugin command with the given name.
171 Parameters
172 ----------
173 runner : click.testing.CliRunner
174 The test runner to use to create the isolated filesystem.
175 commandModule : `str`
176 The importable module that the command can be imported from.
177 commandName : `str`
178 The name of the command being published to import.
179 """
180 with runner.isolated_filesystem():
181 with open("resources.yaml", "w") as f:
182 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
183 # Add a colon to the end of the path on the next line, this tests the
184 # case where the lookup in LoaderCLI._getPluginList generates an empty
185 # string in one of the list entries and verifies that the empty string
186 # is properly stripped out.
187 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
188 yield
191def addArgumentHelp(doc, helpText):
192 """Add a Click argument's help message to a function's documentation.
194 This is needed because click presents arguments in the order the argument
195 decorators are applied to a function, top down. But, the evaluation of the
196 decorators happens bottom up, so if arguments just append their help to the
197 function's docstring, the argument descriptions appear in reverse order
198 from the order they are applied in.
200 Parameters
201 ----------
202 doc : `str`
203 The function's docstring.
204 helpText : `str`
205 The argument's help string to be inserted into the function's
206 docstring.
208 Returns
209 -------
210 doc : `str`
211 Updated function documentation.
212 """
213 if doc is None: 213 ↛ 214line 213 didn't jump to line 214, because the condition on line 213 was never true
214 doc = helpText
215 else:
216 # See click documentation for details:
217 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
218 # In short, text for the click command help can be truncated by putting
219 # "\f" in the docstring, everything after it should be removed
220 if "\f" in doc: 220 ↛ 221line 220 didn't jump to line 221, because the condition on line 220 was never true
221 doc = doc.split("\f")[0]
223 doclines = doc.splitlines()
224 # The function's docstring may span multiple lines, so combine the
225 # docstring from all the first lines until a blank line is encountered.
226 # (Lines after the first blank line will be argument help.)
227 while len(doclines) > 1 and doclines[1]:
228 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
229 # Add standard indent to help text for proper alignment with command
230 # function documentation:
231 helpText = " " + helpText
232 doclines.insert(1, helpText)
233 doclines.insert(1, "\n")
234 doc = "\n".join(doclines)
235 return doc
238def split_commas(context, param, values):
239 """Process a tuple of values, where each value may contain comma-separated
240 values, and return a single list of all the passed-in values.
242 This function can be passed to the 'callback' argument of a click.option to
243 allow it to process comma-separated values (e.g. "--my-opt a,b,c").
245 Parameters
246 ----------
247 context : `click.Context` or `None`
248 The current execution context. Unused, but Click always passes it to
249 callbacks.
250 param : `click.core.Option` or `None`
251 The parameter being handled. Unused, but Click always passes it to
252 callbacks.
253 values : [`str`]
254 All the values passed for this option. Strings may contain commas,
255 which will be treated as delimiters for separate values.
257 Returns
258 -------
259 list of string
260 The passed in values separated by commas and combined into a single
261 list.
262 """
263 if values is None:
264 return values
265 valueList = []
266 for value in ensure_iterable(values):
267 valueList.extend(value.split(","))
268 return tuple(valueList)
271def split_kv(
272 context,
273 param,
274 values,
275 choice=None,
276 multiple=True,
277 normalize=False,
278 separator="=",
279 unseparated_okay=False,
280 return_type=dict,
281 default_key="",
282 reverse_kv=False,
283 add_to_default=False,
284):
285 """Process a tuple of values that are key-value pairs separated by a given
286 separator. Multiple pairs may be comma separated. Return a dictionary of
287 all the passed-in values.
289 This function can be passed to the 'callback' argument of a click.option to
290 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
292 Parameters
293 ----------
294 context : `click.Context` or `None`
295 The current execution context. Unused, but Click always passes it to
296 callbacks.
297 param : `click.core.Option` or `None`
298 The parameter being handled. Unused, but Click always passes it to
299 callbacks.
300 values : [`str`]
301 All the values passed for this option. Strings may contain commas,
302 which will be treated as delimiters for separate values.
303 choice : `click.Choice`, optional
304 If provided, verify each value is a valid choice using the provided
305 `click.Choice` instance. If None, no verification will be done. By
306 default None
307 multiple : `bool`, optional
308 If true, the value may contain multiple comma-separated values. By
309 default True.
310 normalize : `bool`, optional
311 If True and `choice.case_sensitive == False`, normalize the string the
312 user provided to match the choice's case. By default False.
313 separator : str, optional
314 The character that separates key-value pairs. May not be a comma or an
315 empty space (for space separators use Click's default implementation
316 for tuples; `type=(str, str)`). By default "=".
317 unseparated_okay : `bool`, optional
318 If True, allow values that do not have a separator. They will be
319 returned in the values dict as a tuple of values in the key '', that
320 is: `values[''] = (unseparated_values, )`. By default False.
321 return_type : `type`, must be `dict` or `tuple`
322 The type of the value that should be returned.
323 If `dict` then the returned object will be a dict, for each item in
324 values, the value to the left of the separator will be the key and the
325 value to the right of the separator will be the value.
326 If `tuple` then the returned object will be a tuple. Each item in the
327 tuple will be 2-item tuple, the first item will be the value to the
328 left of the separator and the second item will be the value to the
329 right. By default `dict`.
330 default_key : `Any`
331 The key to use if a value is passed that is not a key-value pair.
332 (Passing values that are not key-value pairs requires
333 ``unseparated_okay`` to be `True`.)
334 reverse_kv : bool
335 If true then for each item in values, the value to the left of the
336 separator is treated as the value and the value to the right of the
337 separator is treated as the key. By default False.
338 add_to_default : `bool`, optional
339 If True, then passed-in values will not overwrite the default value
340 unless the ``return_type`` is `dict` and passed-in value(s) have the
341 same key(s) as the default value.
343 Returns
344 -------
345 values : `dict` [`str`, `str`]
346 The passed-in values in dict form.
348 Raises
349 ------
350 `click.ClickException`
351 Raised if the separator is not found in an entry, or if duplicate keys
352 are encountered.
353 """
355 def norm(val):
356 """If `normalize` is True and `choice` is not `None`, find the value
357 in the available choices and return the value as spelled in the
358 choices.
360 Assumes that val exists in choices; `split_kv` uses the `choice`
361 instance to verify val is a valid choice.
362 """
363 if normalize and choice is not None:
364 v = val.casefold()
365 for opt in choice.choices:
366 if opt.casefold() == v:
367 return opt
368 return val
370 class RetDict:
371 def __init__(self):
372 self.ret = {}
374 def add(self, key, val):
375 if reverse_kv:
376 key, val = val, key
377 self.ret[key] = val
379 def get(self):
380 return self.ret
382 class RetTuple:
383 def __init__(self):
384 self.ret = []
386 def add(self, key, val):
387 if reverse_kv:
388 key, val = val, key
389 self.ret.append((key, val))
391 def get(self):
392 return tuple(self.ret)
394 if separator in (",", " "):
395 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
396 vals = values # preserve the original argument for error reporting below.
398 if add_to_default:
399 default = param.get_default(context)
400 if default:
401 vals = itertools.chain(default, vals)
403 if return_type is dict:
404 ret = RetDict()
405 elif return_type is tuple:
406 ret = RetTuple()
407 else:
408 raise click.ClickException(
409 message=f"Internal error: invalid return type '{return_type}' for split_kv."
410 )
411 if multiple:
412 vals = split_commas(context, param, vals)
413 for val in ensure_iterable(vals):
414 if unseparated_okay and separator not in val:
415 if choice is not None:
416 choice(val) # will raise if val is an invalid choice
417 ret.add(default_key, norm(val))
418 else:
419 try:
420 k, v = val.split(separator)
421 if choice is not None:
422 choice(v) # will raise if val is an invalid choice
423 except ValueError:
424 raise click.ClickException(
425 message=f"Could not parse key-value pair '{val}' using separator '{separator}', "
426 f"with multiple values {'allowed' if multiple else 'not allowed'}."
427 )
428 ret.add(k, norm(v))
429 return ret.get()
432def to_upper(context, param, value):
433 """Convert a value to upper case.
435 Parameters
436 ----------
437 context : click.Context
439 values : string
440 The value to be converted.
442 Returns
443 -------
444 string
445 A copy of the passed-in value, converted to upper case.
446 """
447 return value.upper()
450def unwrap(val):
451 """Remove newlines and leading whitespace from a multi-line string with
452 a consistent indentation level.
454 The first line of the string may be only a newline or may contain text
455 followed by a newline, either is ok. After the first line, each line must
456 begin with a consistant amount of whitespace. So, content of a
457 triple-quoted string may begin immediately after the quotes, or the string
458 may start with a newline. Each line after that must be the same amount of
459 indentation/whitespace followed by text and a newline. The last line may
460 end with a new line but is not required to do so.
462 Parameters
463 ----------
464 val : `str`
465 The string to change.
467 Returns
468 -------
469 strippedString : `str`
470 The string with newlines, indentation, and leading and trailing
471 whitespace removed.
472 """
474 def splitSection(val):
475 if not val.startswith("\n"): 475 ↛ 479line 475 didn't jump to line 479, because the condition on line 475 was never false
476 firstLine, _, val = val.partition("\n")
477 firstLine += " "
478 else:
479 firstLine = ""
480 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
482 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
485class option_section: # noqa: N801
486 """Decorator to add a section label between options in the help text of a
487 command.
489 Parameters
490 ----------
491 sectionText : `str`
492 The text to print in the section identifier.
493 """
495 def __init__(self, sectionText):
496 self.sectionText = "\n" + sectionText
498 def __call__(self, f):
499 # Generate a parameter declaration that will be unique for this
500 # section.
501 return click.option(
502 f"--option-section-{str(uuid.uuid4())}", sectionText=self.sectionText, cls=OptionSection
503 )(f)
506class MWPath(click.Path):
507 """Overrides click.Path to implement file-does-not-exist checking.
509 Changes the definition of ``exists` so that `True` indicates the location
510 (file or directory) must exist, `False` indicates the location must *not*
511 exist, and `None` indicates that the file may exist or not. The standard
512 definition for the `click.Path` ``exists`` parameter is that for `True` a
513 location must exist, but `False` means it is not required to exist (not
514 that it is required to not exist).
516 Parameters
517 ----------
518 exists : `True`, `False`, or `None`
519 If `True`, the location (file or directory) indicated by the caller
520 must exist. If `False` the location must not exist. If `None`, the
521 location may exist or not.
523 For other parameters see `click.Path`.
524 """
526 def __init__(
527 self,
528 exists=None,
529 file_okay=True,
530 dir_okay=True,
531 writable=False,
532 readable=True,
533 resolve_path=False,
534 allow_dash=False,
535 path_type=None,
536 ):
537 self.mustNotExist = exists is False
538 if exists is None: 538 ↛ 540line 538 didn't jump to line 540, because the condition on line 538 was never false
539 exists = False
540 super().__init__(
541 exists=exists,
542 file_okay=file_okay,
543 dir_okay=dir_okay,
544 writable=writable,
545 readable=readable,
546 resolve_path=resolve_path,
547 allow_dash=allow_dash,
548 path_type=path_type,
549 )
551 def convert(self, value, param, ctx):
552 """Called by click.ParamType to "convert values through types".
553 `click.Path` uses this step to verify Path conditions."""
554 if self.mustNotExist and os.path.exists(value):
555 self.fail(f'Path "{value}" should not exist.')
556 return super().convert(value, param, ctx)
559class MWOption(click.Option):
560 """Overrides click.Option with desired behaviors."""
562 def make_metavar(self):
563 """Overrides `click.Option.make_metavar`. Makes the metavar for the
564 help menu. Adds a space and an elipsis after the metavar name if
565 the option accepts multiple inputs, otherwise defers to the base
566 implementation.
568 By default click does not add an elipsis when multiple is True and
569 nargs is 1. And when nargs does not equal 1 click adds an elipsis
570 without a space between the metavar and the elipsis, but we prefer a
571 space between.
573 Does not get called for some option types (e.g. flag) so metavar
574 transformation that must apply to all types should be applied in
575 get_help_record.
576 """
577 metavar = super().make_metavar()
578 if self.multiple and self.nargs == 1:
579 metavar += " ..."
580 elif self.nargs != 1:
581 metavar = f"{metavar[:-3]} ..."
582 return metavar
585class MWArgument(click.Argument):
586 """Overrides click.Argument with desired behaviors."""
588 def make_metavar(self):
589 """Overrides `click.Option.make_metavar`. Makes the metavar for the
590 help menu. Always adds a space and an elipsis (' ...') after the
591 metavar name if the option accepts multiple inputs.
593 By default click adds an elipsis without a space between the metavar
594 and the elipsis, but we prefer a space between.
596 Returns
597 -------
598 metavar : `str`
599 The metavar value.
600 """
601 metavar = super().make_metavar()
602 if self.nargs != 1:
603 metavar = f"{metavar[:-3]} ..."
604 return metavar
607class OptionSection(MWOption):
608 """Implements an Option that prints a section label in the help text and
609 does not pass any value to the command function.
611 This class does a bit of hackery to add a section label to a click command
612 help output: first, `expose_value` is set to `False` so that no value is
613 passed to the command function. Second, this class overrides
614 `click.Option.get_help_record` to return the section label string without
615 any prefix so that it stands out as a section label.
617 This class overrides the hidden attribute because our documentation build
618 tool, sphinx-click, implements its own `get_help_record` function which
619 builds the record from other option values (e.g. `name`, `opts`), which
620 breaks the hack we use to make `get_help_record` only return the
621 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
622 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
623 entering its `_get_help_record` function. So, making the hidden property
624 return True hides this option from sphinx-click, while allowing the section
625 text to be returned by our `get_help_record` method when using Click.
627 The intention for this implementation is to do minimally invasive overrides
628 of the click classes so as to be robust and easy to fix if the click
629 internals change.
631 Parameters
632 ----------
633 sectionName : `str`
634 The parameter declaration for this option. It is not shown to the user,
635 it must be unique within the command. If using the `section` decorator
636 to add a section to a command's options, the section name is
637 auto-generated.
638 sectionText : `str`
639 The text to print in the section identifier.
640 """
642 @property
643 def hidden(self):
644 return True
646 @hidden.setter
647 def hidden(self, val):
648 pass
650 def __init__(self, sectionName, sectionText):
651 super().__init__(sectionName, expose_value=False)
652 self.sectionText = sectionText
654 def get_help_record(self, ctx):
655 return (self.sectionText, "")
658class MWOptionDecorator:
659 """Wraps the click.option decorator to enable shared options to be declared
660 and allows inspection of the shared option.
661 """
663 def __init__(self, *param_decls, **kwargs):
664 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption), **kwargs)
665 opt = click.Option(param_decls, **kwargs)
666 self._name = opt.name
667 self._opts = opt.opts
669 def name(self):
670 """Get the name that will be passed to the command function for this
671 option."""
672 return self._name
674 def opts(self):
675 """Get the flags that will be used for this option on the command
676 line."""
677 return self._opts
679 @property
680 def help(self):
681 """Get the help text for this option. Returns an empty string if no
682 help was defined."""
683 return self.partialOpt.keywords.get("help", "")
685 def __call__(self, *args, **kwargs):
686 return self.partialOpt(*args, **kwargs)
689class MWArgumentDecorator:
690 """Wraps the click.argument decorator to enable shared arguments to be
691 declared."""
693 def __init__(self, *param_decls, **kwargs):
694 self._helpText = kwargs.pop("help", None)
695 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
697 def __call__(self, *args, help=None, **kwargs):
698 def decorator(f):
699 if help is not None:
700 self._helpText = help
701 if self._helpText:
702 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
703 return self.partialArg(*args, **kwargs)(f)
705 return decorator
708class MWCommand(click.Command):
709 """Command subclass that stores a copy of the args list for use by the
710 command."""
712 extra_epilog = None
714 def __init__(self, *args, **kwargs):
715 # wrap callback method with catch_and_exit decorator
716 callback = kwargs.get("callback")
717 if callback is not None: 717 ↛ 720line 717 didn't jump to line 720, because the condition on line 717 was never false
718 kwargs = kwargs.copy()
719 kwargs["callback"] = catch_and_exit(callback)
720 super().__init__(*args, **kwargs)
722 def _capture_args(self, ctx, args):
723 """Capture the command line options and arguments.
725 See details about what is captured and the order in which it is stored
726 in the documentation of `MWCtxObj`.
728 Parameters
729 ----------
730 ctx : `click.Context`
731 The current Context.
732 args : `list` [`str`]
733 The list of arguments from the command line, split at spaces but
734 not at separators (like "=").
735 """
736 parser = self.make_parser(ctx)
737 opts, _, param_order = parser.parse_args(args=list(args))
738 # `param_order` is a list of click.Option and click.Argument, there is
739 # one item for each time the Option or Argument was used on the
740 # command line. Options will precede Arguments, within each sublist
741 # they are in the order they were used on the command line. Note that
742 # click.Option and click.Argument do not contain the value from the
743 # command line; values are in `opts`.
744 #
745 # `opts` is a dict where the key is the argument name to the
746 # click.Command function, this name matches the `click.Option.name` or
747 # `click.Argument.name`. For Options, an item will only be present if
748 # the Option was used on the command line. For Arguments, an item will
749 # always be present and if no value was provided on the command line
750 # the value will be `None`. If the option accepts multiple values, the
751 # value in `opts` is a tuple, otherwise it is a single item.
752 next_idx = Counter()
753 captured_args = []
754 for param in param_order:
755 if isinstance(param, click.Option):
756 if param.multiple:
757 val = opts[param.name][next_idx[param.name]]
758 next_idx[param.name] += 1
759 else:
760 val = opts[param.name]
761 if param.is_flag:
762 # Bool options store their True flags in opts and their
763 # False flags in secondary_opts.
764 if val:
765 flag = max(param.opts, key=len)
766 else:
767 flag = max(param.secondary_opts, key=len)
768 captured_args.append(flag)
769 else:
770 captured_args.append(max(param.opts, key=len))
771 captured_args.append(val)
772 elif isinstance(param, click.Argument):
773 if (opt := opts[param.name]) is not None:
774 captured_args.append(opt)
775 else:
776 assert False # All parameters should be an Option or an Argument.
777 MWCtxObj.getFrom(ctx).args = captured_args
779 def parse_args(self, ctx, args):
780 """Given a context and a list of arguments this creates the parser and
781 parses the arguments, then modifies the context as necessary. This is
782 automatically invoked by make_context().
784 This function overrides `click.Command.parse_args`.
786 The call to `_capture_args` in this override stores the arguments
787 (option names, option value, and argument values) that were used by the
788 caller on the command line in the context object. These stored
789 arugments can be used by the command function, e.g. to process options
790 in the order they appeared on the command line (pipetask uses this
791 feature to create pipeline actions in an order from different options).
793 Parameters
794 ----------
795 ctx : `click.core.Context`
796 The current Context.ß
797 args : `list` [`str`]
798 The list of arguments from the command line, split at spaces but
799 not at separators (like "=").
800 """
801 self._capture_args(ctx, args)
802 super().parse_args(ctx, args)
804 @property
805 def epilog(self):
806 """Override the epilog attribute to add extra_epilog (if defined by a
807 subclass) to the end of any epilog provided by a subcommand.
808 """
809 ret = self._epilog if self._epilog else ""
810 if self.extra_epilog:
811 if ret:
812 ret += "\n\n"
813 ret += self.extra_epilog
814 return ret
816 @epilog.setter
817 def epilog(self, val):
818 self._epilog = val
821class ButlerCommand(MWCommand):
822 """Command subclass with butler-command specific overrides."""
824 extra_epilog = "See 'butler --help' for more options."
827class OptionGroup:
828 """Base class for an option group decorator. Requires the option group
829 subclass to have a property called `decorator`."""
831 def __call__(self, f):
832 for decorator in reversed(self.decorators):
833 f = decorator(f)
834 return f
837class MWCtxObj:
838 """Helper object for managing the `click.Context.obj` parameter, allows
839 obj data to be managed in a consistent way.
841 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
842 initialize the obj if needed and return a new or existing `MWCtxObj`.
844 The `args` attribute contains a list of options, option values, and
845 argument values that is similar to the list of arguments and options that
846 were passed in on the command line, with differences noted below:
848 * Option namess and option values are first in the list, and argument
849 values come last. The order of options and option values is preserved
850 within the options. The order of argument values is preserved.
852 * The longest option name is used for the option in the `args` list, e.g.
853 if an option accepts both short and long names "-o / --option" and the
854 short option name "-o" was used on the command line, the longer name will
855 be the one that appears in `args`.
857 * A long option name (which begins with two dashes "--") and its value may
858 be separated by an equal sign; the name and value are split at the equal
859 sign and it is removed. In `args`, the option is in one list item, and
860 the option value (without the equal sign) is in the next list item. e.g.
861 "--option=foo" and "--option foo" both become `["--opt", "foo"]` in
862 `args`.
864 * A short option name, (which begins with one dash "-") and its value are
865 split immediately after the short option name, and if there is
866 whitespace between the short option name and its value it is removed.
867 Everything after the short option name (excluding whitespace) is included
868 in the value. If the `Option` has a long name, the long name will be used
869 in `args` e.g. for the option "-o / --option": "-ofoo" and "-o foo"
870 become `["--option", "foo"]`, and (note!) "-o=foo" will become
871 `["--option", "=foo"]` (because everything after the short option name,
872 except whitespace, is used for the value (as is standard with unix
873 command line tools).
875 Attributes
876 ----------
877 args : `list` [`str`]
878 A list of options, option values, and arguments simialr to those that
879 were passed in on the command line. See comments about captured options
880 & arguments above.
881 """
883 def __init__(self):
885 self.args = None
887 @staticmethod
888 def getFrom(ctx):
889 """If needed, initialize `ctx.obj` with a new `MWCtxObj`, and return
890 the new or already existing `MWCtxObj`."""
891 if ctx.obj is not None:
892 return ctx.obj
893 ctx.obj = MWCtxObj()
894 return ctx.obj
897def yaml_presets(ctx, param, value):
898 """Click callback that reads additional values from the supplied
899 YAML file.
901 Parameters
902 ----------
903 ctx : `click.context`
904 The context for the click operation. Used to extract the subcommand
905 name and translate option & argument names.
906 param : `str`
907 The parameter name.
908 value : `object`
909 The value of the parameter.
910 """
912 def _name_for_option(ctx: click.Context, option: str) -> str:
913 """Use a CLI option name to find the name of the argument to the
914 command function.
916 Parameters
917 ----------
918 ctx : `click.Context`
919 The context for the click operation.
920 option : `str`
921 The option/argument name from the yaml file.
923 Returns
924 -------
925 name : str
926 The name of the argument to use when calling the click.command
927 function, as it should appear in the `ctx.default_map`.
929 Raises
930 ------
931 RuntimeError
932 Raised if the option name from the yaml file does not exist in the
933 command parameters. This catches misspellings and incorrect useage
934 in the yaml file.
935 """
936 for param in ctx.command.params:
937 # Remove leading dashes: they are not used for option names in the
938 # yaml file.
939 if option in [opt.lstrip("-") for opt in param.opts]:
940 return param.name
941 raise RuntimeError(f"'{option}' is not a valid option for {ctx.info_name}")
943 ctx.default_map = ctx.default_map or {}
944 cmd_name = ctx.info_name
945 if value:
946 try:
947 overrides = _read_yaml_presets(value, cmd_name)
948 options = list(overrides.keys())
949 for option in options:
950 name = _name_for_option(ctx, option)
951 if name == option:
952 continue
953 overrides[name] = overrides.pop(option)
954 except Exception as e:
955 raise click.BadOptionUsage(
956 option_name=param.name,
957 message=f"Error reading overrides file: {e}",
958 ctx=ctx,
959 )
960 # Override the defaults for this subcommand
961 ctx.default_map.update(overrides)
962 return
965def _read_yaml_presets(file_uri, cmd_name):
966 """Read file command line overrides from YAML config file.
968 Parameters
969 ----------
970 file_uri : `str`
971 URI of override YAML file containing the command line overrides.
972 They should be grouped by command name.
973 cmd_name : `str`
974 The subcommand name that is being modified.
976 Returns
977 -------
978 overrides : `dict` of [`str`, Any]
979 The relevant command line options read from the override file.
980 """
981 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
982 config = Config(file_uri)
983 return config[cmd_name]
986def sortAstropyTable(table, dimensions, sort_first=None):
987 """Sort an astropy table, with prioritization given to columns in this
988 order:
989 1. the provided named columns
990 2. spatial and temporal columns
991 3. the rest of the columns
993 The table is sorted in-place, and is also returned for convenience.
995 Parameters
996 ----------
997 table : `astropy.table.Table`
998 The table to sort
999 dimensions : `list` [``Dimension``]
1000 The dimensions of the dataIds in the table (the dimensions should be
1001 the same for all the dataIds). Used to determine if the column is
1002 spatial, temporal, or neither.
1003 sort_first : `list` [`str`]
1004 The names of columns that should be sorted first, before spatial and
1005 temporal columns.
1007 Returns
1008 -------
1009 `astropy.table.Table`
1010 For convenience, the table that has been sorted.
1011 """
1012 # For sorting we want to ignore the id
1013 # We also want to move temporal or spatial dimensions earlier
1014 sort_first = sort_first or []
1015 sort_early = []
1016 sort_late = []
1017 for dim in dimensions:
1018 if dim.spatial or dim.temporal:
1019 sort_early.extend(dim.required.names)
1020 else:
1021 sort_late.append(str(dim))
1022 sort_keys = sort_first + sort_early + sort_late
1023 # The required names above means that we have the possibility of
1024 # repeats of sort keys. Now have to remove them
1025 # (order is retained by dict creation).
1026 sort_keys = list(dict.fromkeys(sort_keys).keys())
1028 table.sort(sort_keys)
1029 return table
1032def catch_and_exit(func):
1033 """Decorator which catches all exceptions, prints an exception traceback
1034 and signals click to exit.
1035 """
1037 @wraps(func)
1038 def inner(*args, **kwargs):
1039 try:
1040 func(*args, **kwargs)
1041 except (click.exceptions.ClickException, click.exceptions.Exit, click.exceptions.Abort):
1042 # this is handled by click itself
1043 raise
1044 except Exception:
1045 exc_type, exc_value, exc_tb = sys.exc_info()
1046 if exc_tb.tb_next:
1047 # do not show this decorator in traceback
1048 exc_tb = exc_tb.tb_next
1049 log.exception(
1050 "Caught an exception, details are in traceback:", exc_info=(exc_type, exc_value, exc_tb)
1051 )
1052 # tell click to stop, this never returns.
1053 click.get_current_context().exit(1)
1055 return inner