Coverage for python/lsst/daf/butler/cli/utils.py : 37%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23import click.testing
24from contextlib import contextmanager
25import copy
26from functools import partial
27import itertools
28import logging
29import os
30import textwrap
31import traceback
32from unittest.mock import patch
33import uuid
34import yaml
36from .cliLog import CliLog
37from ..core.utils import iterable
38from ..core.config import Config
40log = logging.getLogger(__name__)
42# This is used as the metavar argument to Options that accept multiple string
43# inputs, which may be comma-separarated. For example:
44# --my-opt foo,bar --my-opt baz.
45# Other arguments to the Option should include multiple=true and
46# callback=split_kv.
47typeStrAcceptsMultiple = "TEXT ..."
48typeStrAcceptsSingle = "TEXT"
50# For parameters that support key-value inputs, this defines the separator
51# for those inputs.
52split_kv_separator = "="
55def astropyTablesToStr(tables):
56 """Render astropy tables to string as they are displayed in the CLI.
58 Output formatting matches ``printAstropyTables``.
59 """
60 ret = ""
61 for table in tables:
62 ret += "\n"
63 table.pformat_all()
64 ret += "\n"
65 return ret
68def printAstropyTables(tables):
69 """Print astropy tables to be displayed in the CLI.
71 Output formatting matches ``astropyTablesToStr``.
72 """
73 for table in tables:
74 print("")
75 table.pprint_all()
76 print("")
79def textTypeStr(multiple):
80 """Get the text type string for CLI help documentation.
82 Parameters
83 ----------
84 multiple : `bool`
85 True if multiple text values are allowed, False if only one value is
86 allowed.
88 Returns
89 -------
90 textTypeStr : `str`
91 The type string to use.
92 """
93 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
96class LogCliRunner(click.testing.CliRunner):
97 """A test runner to use when the logging system will be initialized by code
98 under test, calls CliLog.resetLog(), which undoes any logging setup that
99 was done with the CliLog interface.
101 lsst.log modules can not be set back to an uninitialized state (python
102 logging modules can be set back to NOTSET), instead they are set to
103 `CliLog.defaultLsstLogLevel`."""
105 def invoke(self, *args, **kwargs):
106 result = super().invoke(*args, **kwargs)
107 CliLog.resetLog()
108 return result
111def clickResultMsg(result):
112 """Get a standard assert message from a click result
114 Parameters
115 ----------
116 result : click.Result
117 The result object returned from click.testing.CliRunner.invoke
119 Returns
120 -------
121 msg : `str`
122 The message string.
123 """
124 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
125 if result.exception:
126 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
127 return msg
130@contextmanager
131def command_test_env(runner, commandModule, commandName):
132 """A context manager that creates (and then cleans up) an environment that
133 provides a CLI plugin command with the given name.
135 Parameters
136 ----------
137 runner : click.testing.CliRunner
138 The test runner to use to create the isolated filesystem.
139 commandModule : `str`
140 The importable module that the command can be imported from.
141 commandName : `str`
142 The name of the command being published to import.
143 """
144 with runner.isolated_filesystem():
145 with open("resources.yaml", "w") as f:
146 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
147 # Add a colon to the end of the path on the next line, this tests the
148 # case where the lookup in LoaderCLI._getPluginList generates an empty
149 # string in one of the list entries and verifies that the empty string
150 # is properly stripped out.
151 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
152 yield
155def addArgumentHelp(doc, helpText):
156 """Add a Click argument's help message to a function's documentation.
158 This is needed because click presents arguments in the order the argument
159 decorators are applied to a function, top down. But, the evaluation of the
160 decorators happens bottom up, so if arguments just append their help to the
161 function's docstring, the argument descriptions appear in reverse order
162 from the order they are applied in.
164 Parameters
165 ----------
166 doc : `str`
167 The function's docstring.
168 helpText : `str`
169 The argument's help string to be inserted into the function's
170 docstring.
172 Returns
173 -------
174 doc : `str`
175 Updated function documentation.
176 """
177 if doc is None: 177 ↛ 178line 177 didn't jump to line 178, because the condition on line 177 was never true
178 doc = helpText
179 else:
180 # See click documentation for details:
181 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
182 # In short, text for the click command help can be truncated by putting
183 # "\f" in the docstring, everything after it should be removed
184 if "\f" in doc: 184 ↛ 185line 184 didn't jump to line 185, because the condition on line 184 was never true
185 doc = doc.split("\f")[0]
187 doclines = doc.splitlines()
188 # The function's docstring may span multiple lines, so combine the
189 # docstring from all the first lines until a blank line is encountered.
190 # (Lines after the first blank line will be argument help.)
191 while len(doclines) > 1 and doclines[1]:
192 doclines[0] = " ".join((doclines[0], doclines.pop(1).strip()))
193 doclines.insert(1, helpText)
194 doclines.insert(1, "\n")
195 doc = "\n".join(doclines)
196 return doc
199def split_commas(context, param, values):
200 """Process a tuple of values, where each value may contain comma-separated
201 values, and return a single list of all the passed-in values.
203 This function can be passed to the 'callback' argument of a click.option to
204 allow it to process comma-separated values (e.g. "--my-opt a,b,c").
206 Parameters
207 ----------
208 context : `click.Context` or `None`
209 The current execution context. Unused, but Click always passes it to
210 callbacks.
211 param : `click.core.Option` or `None`
212 The parameter being handled. Unused, but Click always passes it to
213 callbacks.
214 values : [`str`]
215 All the values passed for this option. Strings may contain commas,
216 which will be treated as delimiters for separate values.
218 Returns
219 -------
220 list of string
221 The passed in values separated by commas and combined into a single
222 list.
223 """
224 if values is None:
225 return values
226 valueList = []
227 for value in iterable(values):
228 valueList.extend(value.split(","))
229 return tuple(valueList)
232def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=",
233 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False,
234 add_to_default=False):
235 """Process a tuple of values that are key-value pairs separated by a given
236 separator. Multiple pairs may be comma separated. Return a dictionary of
237 all the passed-in values.
239 This function can be passed to the 'callback' argument of a click.option to
240 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
242 Parameters
243 ----------
244 context : `click.Context` or `None`
245 The current execution context. Unused, but Click always passes it to
246 callbacks.
247 param : `click.core.Option` or `None`
248 The parameter being handled. Unused, but Click always passes it to
249 callbacks.
250 values : [`str`]
251 All the values passed for this option. Strings may contain commas,
252 which will be treated as delimiters for separate values.
253 choice : `click.Choice`, optional
254 If provided, verify each value is a valid choice using the provided
255 `click.Choice` instance. If None, no verification will be done. By
256 default None
257 multiple : `bool`, optional
258 If true, the value may contain multiple comma-separated values. By
259 default True.
260 normalize : `bool`, optional
261 If True and `choice.case_sensitive == False`, normalize the string the
262 user provided to match the choice's case. By default False.
263 separator : str, optional
264 The character that separates key-value pairs. May not be a comma or an
265 empty space (for space separators use Click's default implementation
266 for tuples; `type=(str, str)`). By default "=".
267 unseparated_okay : `bool`, optional
268 If True, allow values that do not have a separator. They will be
269 returned in the values dict as a tuple of values in the key '', that
270 is: `values[''] = (unseparated_values, )`. By default False.
271 return_type : `type`, must be `dict` or `tuple`
272 The type of the value that should be returned.
273 If `dict` then the returned object will be a dict, for each item in
274 values, the value to the left of the separator will be the key and the
275 value to the right of the separator will be the value.
276 If `tuple` then the returned object will be a tuple. Each item in the
277 tuple will be 2-item tuple, the first item will be the value to the
278 left of the separator and the second item will be the value to the
279 right. By default `dict`.
280 default_key : `Any`
281 The key to use if a value is passed that is not a key-value pair.
282 (Passing values that are not key-value pairs requires
283 ``unseparated_okay`` to be `True`.)
284 reverse_kv : bool
285 If true then for each item in values, the value to the left of the
286 separator is treated as the value and the value to the right of the
287 separator is treated as the key. By default False.
288 add_to_default : `bool`, optional
289 If True, then passed-in values will not overwrite the default value
290 unless the ``return_type`` is `dict` and passed-in value(s) have the
291 same key(s) as the default value.
293 Returns
294 -------
295 values : `dict` [`str`, `str`]
296 The passed-in values in dict form.
298 Raises
299 ------
300 `click.ClickException`
301 Raised if the separator is not found in an entry, or if duplicate keys
302 are encountered.
303 """
305 def norm(val):
306 """If `normalize` is True and `choice` is not `None`, find the value
307 in the available choices and return the value as spelled in the
308 choices.
310 Assumes that val exists in choices; `split_kv` uses the `choice`
311 instance to verify val is a valid choice.
312 """
313 if normalize and choice is not None:
314 v = val.casefold()
315 for opt in choice.choices:
316 if opt.casefold() == v:
317 return opt
318 return val
320 class RetDict:
322 def __init__(self):
323 self.ret = {}
325 def add(self, key, val):
326 if reverse_kv:
327 key, val = val, key
328 self.ret[key] = val
330 def get(self):
331 return self.ret
333 class RetTuple:
335 def __init__(self):
336 self.ret = []
338 def add(self, key, val):
339 if reverse_kv:
340 key, val = val, key
341 self.ret.append((key, val))
343 def get(self):
344 return tuple(self.ret)
346 if separator in (",", " "):
347 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
348 vals = values # preserve the original argument for error reporting below.
350 if add_to_default:
351 default = param.get_default(context)
352 if default:
353 vals = itertools.chain(default, vals)
355 if return_type is dict:
356 ret = RetDict()
357 elif return_type is tuple:
358 ret = RetTuple()
359 else:
360 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.")
361 if multiple:
362 vals = split_commas(context, param, vals)
363 for val in iterable(vals):
364 if unseparated_okay and separator not in val:
365 if choice is not None:
366 choice(val) # will raise if val is an invalid choice
367 ret.add(default_key, norm(val))
368 else:
369 try:
370 k, v = val.split(separator)
371 if choice is not None:
372 choice(v) # will raise if val is an invalid choice
373 except ValueError:
374 raise click.ClickException(
375 f"Could not parse key-value pair '{val}' using separator '{separator}', "
376 f"with multiple values {'allowed' if multiple else 'not allowed'}.")
377 ret.add(k, norm(v))
378 return ret.get()
381def to_upper(context, param, value):
382 """Convert a value to upper case.
384 Parameters
385 ----------
386 context : click.Context
388 values : string
389 The value to be converted.
391 Returns
392 -------
393 string
394 A copy of the passed-in value, converted to upper case.
395 """
396 return value.upper()
399def unwrap(val):
400 """Remove newlines and leading whitespace from a multi-line string with
401 a consistent indentation level.
403 The first line of the string may be only a newline or may contain text
404 followed by a newline, either is ok. After the first line, each line must
405 begin with a consistant amount of whitespace. So, content of a
406 triple-quoted string may begin immediately after the quotes, or the string
407 may start with a newline. Each line after that must be the same amount of
408 indentation/whitespace followed by text and a newline. The last line may
409 end with a new line but is not required to do so.
411 Parameters
412 ----------
413 val : `str`
414 The string to change.
416 Returns
417 -------
418 strippedString : `str`
419 The string with newlines, indentation, and leading and trailing
420 whitespace removed.
421 """
422 def splitSection(val):
423 if not val.startswith("\n"): 423 ↛ 427line 423 didn't jump to line 427, because the condition on line 423 was never false
424 firstLine, _, val = val.partition("\n")
425 firstLine += " "
426 else:
427 firstLine = ""
428 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
430 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
433class option_section: # noqa: N801
434 """Decorator to add a section label between options in the help text of a
435 command.
437 Parameters
438 ----------
439 sectionText : `str`
440 The text to print in the section identifier.
441 """
443 def __init__(self, sectionText):
444 self.sectionText = "\n" + sectionText
446 def __call__(self, f):
447 # Generate a parameter declaration that will be unique for this
448 # section.
449 return click.option(f"--option-section-{str(uuid.uuid4())}",
450 sectionText=self.sectionText,
451 cls=OptionSection)(f)
454class MWPath(click.Path):
455 """Overrides click.Path to implement file-does-not-exist checking.
457 Changes the definition of ``exists` so that `True` indicates the location
458 (file or directory) must exist, `False` indicates the location must *not*
459 exist, and `None` indicates that the file may exist or not. The standard
460 definition for the `click.Path` ``exists`` parameter is that for `True` a
461 location must exist, but `False` means it is not required to exist (not
462 that it is required to not exist).
464 Parameters
465 ----------
466 exists : `True`, `False`, or `None`
467 If `True`, the location (file or directory) indicated by the caller
468 must exist. If `False` the location must not exist. If `None`, the
469 location may exist or not.
471 For other parameters see `click.Path`.
472 """
474 def __init__(self, exists=None, file_okay=True, dir_okay=True,
475 writable=False, readable=True, resolve_path=False,
476 allow_dash=False, path_type=None):
477 self.mustNotExist = exists is False
478 if exists is None:
479 exists = False
480 super().__init__(exists, file_okay, dir_okay, writable, readable,
481 resolve_path, allow_dash, path_type)
483 def convert(self, value, param, ctx):
484 """Called by click.ParamType to "convert values through types".
485 `click.Path` uses this step to verify Path conditions."""
486 if self.mustNotExist and os.path.exists(value):
487 self.fail(f'{self.path_type} "{value}" should not exist.')
488 return super().convert(value, param, ctx)
491class MWOption(click.Option):
492 """Overrides click.Option with desired behaviors."""
494 def make_metavar(self):
495 """Overrides `click.Option.make_metavar`. Makes the metavar for the
496 help menu. Adds a space and an elipsis after the metavar name if
497 the option accepts multiple inputs, otherwise defers to the base
498 implementation.
500 By default click does not add an elipsis when multiple is True and
501 nargs is 1. And when nargs does not equal 1 click adds an elipsis
502 without a space between the metavar and the elipsis, but we prefer a
503 space between.
505 Does not get called for some option types (e.g. flag) so metavar
506 transformation that must apply to all types should be applied in
507 get_help_record.
508 """
509 metavar = super().make_metavar()
510 if self.multiple and self.nargs == 1:
511 metavar += " ..."
512 elif self.nargs != 1:
513 metavar = f"{metavar[:-3]} ..."
514 return metavar
517class MWArgument(click.Argument):
518 """Overrides click.Argument with desired behaviors."""
520 def make_metavar(self):
521 """Overrides `click.Option.make_metavar`. Makes the metavar for the
522 help menu. Always adds a space and an elipsis (' ...') after the
523 metavar name if the option accepts multiple inputs.
525 By default click adds an elipsis without a space between the metavar
526 and the elipsis, but we prefer a space between.
528 Returns
529 -------
530 metavar : `str`
531 The metavar value.
532 """
533 metavar = super().make_metavar()
534 if self.nargs != 1:
535 metavar = f"{metavar[:-3]} ..."
536 return metavar
539class OptionSection(MWOption):
540 """Implements an Option that prints a section label in the help text and
541 does not pass any value to the command function.
543 This class does a bit of hackery to add a section label to a click command
544 help output: first, `expose_value` is set to `False` so that no value is
545 passed to the command function. Second, this class overrides
546 `click.Option.get_help_record` to return the section label string without
547 any prefix so that it stands out as a section label.
549 This class overrides the hidden attribute because our documentation build
550 tool, sphinx-click, implements its own `get_help_record` function which
551 builds the record from other option values (e.g. `name`, `opts`), which
552 breaks the hack we use to make `get_help_record` only return the
553 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
554 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
555 entering its `_get_help_record` function. So, making the hidden property
556 return True hides this option from sphinx-click, while allowing the section
557 text to be returned by our `get_help_record` method when using Click.
559 The intention for this implementation is to do minimally invasive overrides
560 of the click classes so as to be robust and easy to fix if the click
561 internals change.
563 Parameters
564 ----------
565 sectionName : `str`
566 The parameter declaration for this option. It is not shown to the user,
567 it must be unique within the command. If using the `section` decorator
568 to add a section to a command's options, the section name is
569 auto-generated.
570 sectionText : `str`
571 The text to print in the section identifier.
572 """
574 @property
575 def hidden(self):
576 return True
578 @hidden.setter
579 def hidden(self, val):
580 pass
582 def __init__(self, sectionName, sectionText):
583 super().__init__(sectionName, expose_value=False)
584 self.sectionText = sectionText
586 def get_help_record(self, ctx):
587 return (self.sectionText, "")
590class MWOptionDecorator:
591 """Wraps the click.option decorator to enable shared options to be declared
592 and allows inspection of the shared option.
593 """
595 def __init__(self, *param_decls, **kwargs):
596 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption),
597 **kwargs)
598 opt = click.Option(param_decls, **kwargs)
599 self._name = opt.name
600 self._opts = opt.opts
602 def name(self):
603 """Get the name that will be passed to the command function for this
604 option."""
605 return self._name
607 def opts(self):
608 """Get the flags that will be used for this option on the command
609 line."""
610 return self._opts
612 @property
613 def help(self):
614 """Get the help text for this option. Returns an empty string if no
615 help was defined."""
616 return self.partialOpt.keywords.get("help", "")
618 def __call__(self, *args, **kwargs):
619 return self.partialOpt(*args, **kwargs)
622class MWArgumentDecorator:
623 """Wraps the click.argument decorator to enable shared arguments to be
624 declared. """
626 def __init__(self, *param_decls, **kwargs):
627 self._helpText = kwargs.pop("help", None)
628 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
630 def __call__(self, *args, help=None, **kwargs):
631 def decorator(f):
632 if help is not None:
633 self._helpText = help
634 if self._helpText: 634 ↛ 636line 634 didn't jump to line 636, because the condition on line 634 was never false
635 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
636 return self.partialArg(*args, **kwargs)(f)
637 return decorator
640class MWCommand(click.Command):
641 """Command subclass that stores a copy of the args list for use by the
642 command."""
644 extra_epilog = None
646 def parse_args(self, ctx, args):
647 MWCtxObj.getFrom(ctx).args = copy.copy(args)
648 super().parse_args(ctx, args)
650 @property
651 def epilog(self):
652 """Override the epilog attribute to add extra_epilog (if defined by a
653 subclass) to the end of any epilog provided by a subcommand.
654 """
655 ret = self._epilog if self._epilog else ""
656 if self.extra_epilog:
657 if ret:
658 ret += "\n\n"
659 ret += self.extra_epilog
660 return ret
662 @epilog.setter
663 def epilog(self, val):
664 self._epilog = val
667class ButlerCommand(MWCommand):
668 """Command subclass with butler-command specific overrides."""
670 extra_epilog = "See 'butler --help' for more options."
673class MWCtxObj():
674 """Helper object for managing the `click.Context.obj` parameter, allows
675 obj data to be managed in a consistent way.
677 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
678 initialize the obj if needed and return a new or existing MWCtxObj.
680 Attributes
681 ----------
682 args : `list` [`str`]
683 The list of arguments (argument values, option flags, and option
684 values), split using whitespace, that were passed in on the command
685 line for the subcommand represented by the parent context object.
686 """
688 def __init__(self):
690 self.args = None
692 @staticmethod
693 def getFrom(ctx):
694 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the
695 new or already existing MWCtxObj."""
696 if ctx.obj is not None:
697 return ctx.obj
698 ctx.obj = MWCtxObj()
699 return ctx.obj
702def yaml_presets(ctx, param, value):
703 """Click callback that reads additional values from the supplied
704 YAML file.
706 Parameters
707 ----------
708 ctx : `click.context`
709 The context for the click operation. Used to extract the subcommand
710 name.
711 param : `str`
712 The parameter name.
713 value : `object`
714 The value of the parameter.
715 """
716 ctx.default_map = ctx.default_map or {}
717 cmd_name = ctx.info_name
718 if value:
719 try:
720 overrides = _read_yaml_presets(value, cmd_name)
721 except Exception as e:
722 raise click.BadOptionUsage(param.name, f"Error reading overrides file: {e}", ctx)
723 # Override the defaults for this subcommand
724 ctx.default_map.update(overrides)
725 return
728def _read_yaml_presets(file_uri, cmd_name):
729 """Read file command line overrides from YAML config file.
731 Parameters
732 ----------
733 file_uri : `str`
734 URI of override YAML file containing the command line overrides.
735 They should be grouped by command name.
736 cmd_name : `str`
737 The subcommand name that is being modified.
739 Returns
740 -------
741 overrides : `dict` of [`str`, Any]
742 The relevant command line options read from the override file.
743 """
744 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
745 config = Config(file_uri)
746 return config[cmd_name]
749def sortAstropyTable(table, dimensions, sort_first=None):
750 """Sort an astropy table, with prioritization given to columns in this
751 order:
752 1. the provided named columns
753 2. spatial and temporal columns
754 3. the rest of the columns
756 The table is sorted in-place, and is also returned for convenience.
758 Parameters
759 ----------
760 table : `astropy.table.Table`
761 The table to sort
762 dimensions : `list` [``Dimension``]
763 The dimensions of the dataIds in the table (the dimensions should be
764 the same for all the dataIds). Used to determine if the column is
765 spatial, temporal, or neither.
766 sort_first : `list` [`str`]
767 The names of columns that should be sorted first, before spatial and
768 temporal columns.
770 Returns
771 -------
772 `astropy.table.Table`
773 For convenience, the table that has been sorted.
774 """
775 # For sorting we want to ignore the id
776 # We also want to move temporal or spatial dimensions earlier
777 sort_first = sort_first or []
778 sort_early = []
779 sort_late = []
780 for dim in dimensions:
781 if dim.spatial or dim.temporal:
782 sort_early.extend(dim.required.names)
783 else:
784 sort_late.append(str(dim))
785 sort_keys = sort_first + sort_early + sort_late
786 # The required names above means that we have the possibility of
787 # repeats of sort keys. Now have to remove them
788 # (order is retained by dict creation).
789 sort_keys = list(dict.fromkeys(sort_keys).keys())
791 table.sort(sort_keys)
792 return table