Coverage for python/lsst/daf/butler/cli/utils.py : 36%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23import click.testing
24from contextlib import contextmanager
25import copy
26from functools import partial
27import itertools
28import logging
29import os
30import textwrap
31import traceback
32from unittest.mock import patch
33import uuid
34import yaml
36from .cliLog import CliLog
37from ..core.utils import iterable
38from ..core.config import Config
40log = logging.getLogger(__name__)
42# This is used as the metavar argument to Options that accept multiple string
43# inputs, which may be comma-separarated. For example:
44# --my-opt foo,bar --my-opt baz.
45# Other arguments to the Option should include multiple=true and
46# callback=split_kv.
47typeStrAcceptsMultiple = "TEXT ..."
48typeStrAcceptsSingle = "TEXT"
50# For parameters that support key-value inputs, this defines the separator
51# for those inputs.
52split_kv_separator = "="
55def textTypeStr(multiple):
56 """Get the text type string for CLI help documentation.
58 Parameters
59 ----------
60 multiple : `bool`
61 True if multiple text values are allowed, False if only one value is
62 allowed.
64 Returns
65 -------
66 textTypeStr : `str`
67 The type string to use.
68 """
69 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
72class LogCliRunner(click.testing.CliRunner):
73 """A test runner to use when the logging system will be initialized by code
74 under test, calls CliLog.resetLog(), which undoes any logging setup that
75 was done with the CliLog interface.
77 lsst.log modules can not be set back to an uninitialized state (python
78 logging modules can be set back to NOTSET), instead they are set to
79 `CliLog.defaultLsstLogLevel`."""
81 def invoke(self, *args, **kwargs):
82 result = super().invoke(*args, **kwargs)
83 CliLog.resetLog()
84 return result
87def clickResultMsg(result):
88 """Get a standard assert message from a click result
90 Parameters
91 ----------
92 result : click.Result
93 The result object returned from click.testing.CliRunner.invoke
95 Returns
96 -------
97 msg : `str`
98 The message string.
99 """
100 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
101 if result.exception:
102 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
103 return msg
106@contextmanager
107def command_test_env(runner, commandModule, commandName):
108 """A context manager that creates (and then cleans up) an environment that
109 provides a CLI plugin command with the given name.
111 Parameters
112 ----------
113 runner : click.testing.CliRunner
114 The test runner to use to create the isolated filesystem.
115 commandModule : `str`
116 The importable module that the command can be imported from.
117 commandName : `str`
118 The name of the command being published to import.
119 """
120 with runner.isolated_filesystem():
121 with open("resources.yaml", "w") as f:
122 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
123 # Add a colon to the end of the path on the next line, this tests the
124 # case where the lookup in LoaderCLI._getPluginList generates an empty
125 # string in one of the list entries and verifies that the empty string
126 # is properly stripped out.
127 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
128 yield
131def addArgumentHelp(doc, helpText):
132 """Add a Click argument's help message to a function's documentation.
134 This is needed because click presents arguments in the order the argument
135 decorators are applied to a function, top down. But, the evaluation of the
136 decorators happens bottom up, so if arguments just append their help to the
137 function's docstring, the argument descriptions appear in reverse order
138 from the order they are applied in.
140 Parameters
141 ----------
142 doc : `str`
143 The function's docstring.
144 helpText : `str`
145 The argument's help string to be inserted into the function's
146 docstring.
148 Returns
149 -------
150 doc : `str`
151 Updated function documentation.
152 """
153 if doc is None: 153 ↛ 154line 153 didn't jump to line 154, because the condition on line 153 was never true
154 doc = helpText
155 else:
156 # See click documentation for details:
157 # https://click.palletsprojects.com/en/7.x/documentation/#truncating-help-texts
158 # In short, text for the click command help can be truncated by putting
159 # "\f" in the docstring, everything after it should be removed
160 if "\f" in doc: 160 ↛ 161line 160 didn't jump to line 161, because the condition on line 160 was never true
161 doc = doc.split("\f")[0]
163 doclines = doc.splitlines()
164 doclines.insert(1, helpText)
165 doclines.insert(1, "\n")
166 doc = "\n".join(doclines)
167 return doc
170def split_commas(context, param, values):
171 """Process a tuple of values, where each value may contain comma-separated
172 values, and return a single list of all the passed-in values.
174 This function can be passed to the 'callback' argument of a click.option to
175 allow it to process comma-separated values (e.g. "--my-opt a,b,c").
177 Parameters
178 ----------
179 context : `click.Context` or `None`
180 The current execution context. Unused, but Click always passes it to
181 callbacks.
182 param : `click.core.Option` or `None`
183 The parameter being handled. Unused, but Click always passes it to
184 callbacks.
185 values : [`str`]
186 All the values passed for this option. Strings may contain commas,
187 which will be treated as delimiters for separate values.
189 Returns
190 -------
191 list of string
192 The passed in values separated by commas and combined into a single
193 list.
194 """
195 if values is None:
196 return values
197 valueList = []
198 for value in iterable(values):
199 valueList.extend(value.split(","))
200 return tuple(valueList)
203def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=",
204 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False,
205 add_to_default=False):
206 """Process a tuple of values that are key-value pairs separated by a given
207 separator. Multiple pairs may be comma separated. Return a dictionary of
208 all the passed-in values.
210 This function can be passed to the 'callback' argument of a click.option to
211 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
213 Parameters
214 ----------
215 context : `click.Context` or `None`
216 The current execution context. Unused, but Click always passes it to
217 callbacks.
218 param : `click.core.Option` or `None`
219 The parameter being handled. Unused, but Click always passes it to
220 callbacks.
221 values : [`str`]
222 All the values passed for this option. Strings may contain commas,
223 which will be treated as delimiters for separate values.
224 choice : `click.Choice`, optional
225 If provided, verify each value is a valid choice using the provided
226 `click.Choice` instance. If None, no verification will be done. By
227 default None
228 multiple : `bool`, optional
229 If true, the value may contain multiple comma-separated values. By
230 default True.
231 normalize : `bool`, optional
232 If True and `choice.case_sensitive == False`, normalize the string the
233 user provided to match the choice's case. By default False.
234 separator : str, optional
235 The character that separates key-value pairs. May not be a comma or an
236 empty space (for space separators use Click's default implementation
237 for tuples; `type=(str, str)`). By default "=".
238 unseparated_okay : `bool`, optional
239 If True, allow values that do not have a separator. They will be
240 returned in the values dict as a tuple of values in the key '', that
241 is: `values[''] = (unseparated_values, )`. By default False.
242 return_type : `type`, must be `dict` or `tuple`
243 The type of the value that should be returned.
244 If `dict` then the returned object will be a dict, for each item in
245 values, the value to the left of the separator will be the key and the
246 value to the right of the separator will be the value.
247 If `tuple` then the returned object will be a tuple. Each item in the
248 tuple will be 2-item tuple, the first item will be the value to the
249 left of the separator and the second item will be the value to the
250 right. By default `dict`.
251 default_key : `Any`
252 The key to use if a value is passed that is not a key-value pair.
253 (Passing values that are not key-value pairs requires
254 ``unseparated_okay`` to be `True`.)
255 reverse_kv : bool
256 If true then for each item in values, the value to the left of the
257 separator is treated as the value and the value to the right of the
258 separator is treated as the key. By default False.
259 add_to_default : `bool`, optional
260 If True, then passed-in values will not overwrite the default value
261 unless the ``return_type`` is `dict` and passed-in value(s) have the
262 same key(s) as the default value.
264 Returns
265 -------
266 values : `dict` [`str`, `str`]
267 The passed-in values in dict form.
269 Raises
270 ------
271 `click.ClickException`
272 Raised if the separator is not found in an entry, or if duplicate keys
273 are encountered.
274 """
276 def norm(val):
277 """If `normalize` is True and `choice` is not `None`, find the value
278 in the available choices and return the value as spelled in the
279 choices.
281 Assumes that val exists in choices; `split_kv` uses the `choice`
282 instance to verify val is a valid choice.
283 """
284 if normalize and choice is not None:
285 v = val.casefold()
286 for opt in choice.choices:
287 if opt.casefold() == v:
288 return opt
289 return val
291 class RetDict:
293 def __init__(self):
294 self.ret = {}
296 def add(self, key, val):
297 if reverse_kv:
298 key, val = val, key
299 self.ret[key] = val
301 def get(self):
302 return self.ret
304 class RetTuple:
306 def __init__(self):
307 self.ret = []
309 def add(self, key, val):
310 if reverse_kv:
311 key, val = val, key
312 self.ret.append((key, val))
314 def get(self):
315 return tuple(self.ret)
317 if separator in (",", " "):
318 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
319 vals = values # preserve the original argument for error reporting below.
321 if add_to_default:
322 default = param.get_default(context)
323 if default:
324 vals = itertools.chain(default, vals)
326 if return_type is dict:
327 ret = RetDict()
328 elif return_type is tuple:
329 ret = RetTuple()
330 else:
331 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.")
332 if multiple:
333 vals = split_commas(context, param, vals)
334 for val in iterable(vals):
335 if unseparated_okay and separator not in val:
336 if choice is not None:
337 choice(val) # will raise if val is an invalid choice
338 ret.add(default_key, norm(val))
339 else:
340 try:
341 k, v = val.split(separator)
342 if choice is not None:
343 choice(v) # will raise if val is an invalid choice
344 except ValueError:
345 raise click.ClickException(
346 f"Could not parse key-value pair '{val}' using separator '{separator}', "
347 f"with multiple values {'allowed' if multiple else 'not allowed'}.")
348 ret.add(k, norm(v))
349 return ret.get()
352def to_upper(context, param, value):
353 """Convert a value to upper case.
355 Parameters
356 ----------
357 context : click.Context
359 values : string
360 The value to be converted.
362 Returns
363 -------
364 string
365 A copy of the passed-in value, converted to upper case.
366 """
367 return value.upper()
370def unwrap(val):
371 """Remove newlines and leading whitespace from a multi-line string with
372 a consistent indentation level.
374 The first line of the string may be only a newline or may contain text
375 followed by a newline, either is ok. After the first line, each line must
376 begin with a consistant amount of whitespace. So, content of a
377 triple-quoted string may begin immediately after the quotes, or the string
378 may start with a newline. Each line after that must be the same amount of
379 indentation/whitespace followed by text and a newline. The last line may
380 end with a new line but is not required to do so.
382 Parameters
383 ----------
384 val : `str`
385 The string to change.
387 Returns
388 -------
389 strippedString : `str`
390 The string with newlines, indentation, and leading and trailing
391 whitespace removed.
392 """
393 def splitSection(val):
394 if not val.startswith("\n"): 394 ↛ 398line 394 didn't jump to line 398, because the condition on line 394 was never false
395 firstLine, _, val = val.partition("\n")
396 firstLine += " "
397 else:
398 firstLine = ""
399 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
401 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
404class option_section: # noqa: N801
405 """Decorator to add a section label between options in the help text of a
406 command.
408 Parameters
409 ----------
410 sectionText : `str`
411 The text to print in the section identifier.
412 """
414 def __init__(self, sectionText):
415 self.sectionText = "\n" + sectionText
417 def __call__(self, f):
418 # Generate a parameter declaration that will be unique for this
419 # section.
420 return click.option(f"--option-section-{str(uuid.uuid4())}",
421 sectionText=self.sectionText,
422 cls=OptionSection)(f)
425class MWPath(click.Path):
426 """Overrides click.Path to implement file-does-not-exist checking.
428 Changes the definition of ``exists` so that `True` indicates the location
429 (file or directory) must exist, `False` indicates the location must *not*
430 exist, and `None` indicates that the file may exist or not. The standard
431 definition for the `click.Path` ``exists`` parameter is that for `True` a
432 location must exist, but `False` means it is not required to exist (not
433 that it is required to not exist).
435 Parameters
436 ----------
437 exists : `True`, `False`, or `None`
438 If `True`, the location (file or directory) indicated by the caller
439 must exist. If `False` the location must not exist. If `None`, the
440 location may exist or not.
442 For other parameters see `click.Path`.
443 """
445 def __init__(self, exists=None, file_okay=True, dir_okay=True,
446 writable=False, readable=True, resolve_path=False,
447 allow_dash=False, path_type=None):
448 self.mustNotExist = exists is False
449 if exists is None:
450 exists = False
451 super().__init__(exists, file_okay, dir_okay, writable, readable,
452 resolve_path, allow_dash, path_type)
454 def convert(self, value, param, ctx):
455 """Called by click.ParamType to "convert values through types".
456 `click.Path` uses this step to verify Path conditions."""
457 if self.mustNotExist and os.path.exists(value):
458 self.fail(f'{self.path_type} "{value}" should not exist.')
459 return super().convert(value, param, ctx)
462class MWOption(click.Option):
463 """Overrides click.Option with desired behaviors."""
465 def make_metavar(self):
466 """Overrides `click.Option.make_metavar`. Makes the metavar for the
467 help menu. Adds a space and an elipsis after the metavar name if
468 the option accepts multiple inputs, otherwise defers to the base
469 implementation.
471 By default click does not add an elipsis when multiple is True and
472 nargs is 1. And when nargs does not equal 1 click adds an elipsis
473 without a space between the metavar and the elipsis, but we prefer a
474 space between.
476 Does not get called for some option types (e.g. flag) so metavar
477 transformation that must apply to all types should be applied in
478 get_help_record.
479 """
480 metavar = super().make_metavar()
481 if self.multiple and self.nargs == 1:
482 metavar += " ..."
483 elif self.nargs != 1:
484 metavar = f"{metavar[:-3]} ..."
485 return metavar
488class MWArgument(click.Argument):
489 """Overrides click.Argument with desired behaviors."""
491 def make_metavar(self):
492 """Overrides `click.Option.make_metavar`. Makes the metavar for the
493 help menu. Always adds a space and an elipsis (' ...') after the
494 metavar name if the option accepts multiple inputs.
496 By default click adds an elipsis without a space between the metavar
497 and the elipsis, but we prefer a space between.
499 Returns
500 -------
501 metavar : `str`
502 The metavar value.
503 """
504 metavar = super().make_metavar()
505 if self.nargs != 1:
506 metavar = f"{metavar[:-3]} ..."
507 return metavar
510class OptionSection(MWOption):
511 """Implements an Option that prints a section label in the help text and
512 does not pass any value to the command function.
514 This class does a bit of hackery to add a section label to a click command
515 help output: first, `expose_value` is set to `False` so that no value is
516 passed to the command function. Second, this class overrides
517 `click.Option.get_help_record` to return the section label string without
518 any prefix so that it stands out as a section label.
520 This class overrides the hidden attribute because our documentation build
521 tool, sphinx-click, implements its own `get_help_record` function which
522 builds the record from other option values (e.g. `name`, `opts`), which
523 breaks the hack we use to make `get_help_record` only return the
524 `sectionText`. Fortunately, Click gets the value of `hidden` inside the
525 `Option`'s `get_help_record`, and `sphinx-click` calls `opt.hidden` before
526 entering its `_get_help_record` function. So, making the hidden property
527 return True hides this option from sphinx-click, while allowing the section
528 text to be returned by our `get_help_record` method when using Click.
530 The intention for this implementation is to do minimally invasive overrides
531 of the click classes so as to be robust and easy to fix if the click
532 internals change.
534 Parameters
535 ----------
536 sectionName : `str`
537 The parameter declaration for this option. It is not shown to the user,
538 it must be unique within the command. If using the `section` decorator
539 to add a section to a command's options, the section name is
540 auto-generated.
541 sectionText : `str`
542 The text to print in the section identifier.
543 """
545 @property
546 def hidden(self):
547 return True
549 @hidden.setter
550 def hidden(self, val):
551 pass
553 def __init__(self, sectionName, sectionText):
554 super().__init__(sectionName, expose_value=False)
555 self.sectionText = sectionText
557 def get_help_record(self, ctx):
558 return (self.sectionText, "")
561class MWOptionDecorator:
562 """Wraps the click.option decorator to enable shared options to be declared
563 and allows inspection of the shared option.
564 """
566 def __init__(self, *param_decls, **kwargs):
567 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption),
568 **kwargs)
569 opt = click.Option(param_decls, **kwargs)
570 self._name = opt.name
571 self._opts = opt.opts
573 def name(self):
574 """Get the name that will be passed to the command function for this
575 option."""
576 return self._name
578 def opts(self):
579 """Get the flags that will be used for this option on the command
580 line."""
581 return self._opts
583 @property
584 def help(self):
585 """Get the help text for this option. Returns an empty string if no
586 help was defined."""
587 return self.partialOpt.keywords.get("help", "")
589 def __call__(self, *args, **kwargs):
590 return self.partialOpt(*args, **kwargs)
593class MWArgumentDecorator:
594 """Wraps the click.argument decorator to enable shared arguments to be
595 declared. """
597 def __init__(self, *param_decls, **kwargs):
598 self._helpText = kwargs.pop("help", None)
599 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
601 def __call__(self, *args, help=None, **kwargs):
602 def decorator(f):
603 if help is not None:
604 self._helpText = help
605 if self._helpText: 605 ↛ 607line 605 didn't jump to line 607, because the condition on line 605 was never false
606 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
607 return self.partialArg(*args, **kwargs)(f)
608 return decorator
611class MWCommand(click.Command):
612 """Command subclass that stores a copy of the args list for use by the
613 command."""
615 extra_epilog = None
617 def parse_args(self, ctx, args):
618 MWCtxObj.getFrom(ctx).args = copy.copy(args)
619 super().parse_args(ctx, args)
621 @property
622 def epilog(self):
623 """Override the epilog attribute to add extra_epilog (if defined by a
624 subclass) to the end of any epilog provided by a subcommand.
625 """
626 ret = self._epilog if self._epilog else ""
627 if self.extra_epilog:
628 if ret:
629 ret += "\n\n"
630 ret += self.extra_epilog
631 return ret
633 @epilog.setter
634 def epilog(self, val):
635 self._epilog = val
638class ButlerCommand(MWCommand):
639 """Command subclass with butler-command specific overrides."""
641 extra_epilog = "See 'butler --help' for more options."
644class MWCtxObj():
645 """Helper object for managing the `click.Context.obj` parameter, allows
646 obj data to be managed in a consistent way.
648 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
649 initialize the obj if needed and return a new or existing MWCtxObj.
651 Attributes
652 ----------
653 args : `list` [`str`]
654 The list of arguments (argument values, option flags, and option
655 values), split using whitespace, that were passed in on the command
656 line for the subcommand represented by the parent context object.
657 """
659 def __init__(self):
661 self.args = None
663 @staticmethod
664 def getFrom(ctx):
665 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the
666 new or already existing MWCtxObj."""
667 if ctx.obj is not None:
668 return ctx.obj
669 ctx.obj = MWCtxObj()
670 return ctx.obj
673def yaml_presets(ctx, param, value):
674 """Click callback that reads additional values from the supplied
675 YAML file.
677 Parameters
678 ----------
679 ctx : `click.context`
680 The context for the click operation. Used to extract the subcommand
681 name.
682 param : `str`
683 The parameter name.
684 value : `object`
685 The value of the parameter.
686 """
687 ctx.default_map = ctx.default_map or {}
688 cmd_name = ctx.info_name
689 if value:
690 try:
691 overrides = _read_yaml_presets(value, cmd_name)
692 except Exception as e:
693 raise click.BadOptionUsage(param.name, f"Error reading overrides file: {e}", ctx)
694 # Override the defaults for this subcommand
695 ctx.default_map.update(overrides)
696 return
699def _read_yaml_presets(file_uri, cmd_name):
700 """Read file command line overrides from YAML config file.
702 Parameters
703 ----------
704 file_uri : `str`
705 URI of override YAML file containing the command line overrides.
706 They should be grouped by command name.
707 cmd_name : `str`
708 The subcommand name that is being modified.
710 Returns
711 -------
712 overrides : `dict` of [`str`, Any]
713 The relevant command line options read from the override file.
714 """
715 log.debug("Reading command line overrides for subcommand %s from URI %s", cmd_name, file_uri)
716 config = Config(file_uri)
717 return config[cmd_name]
720def sortAstropyTable(table, dimensions, sort_first=None):
721 """Sort an astropy table, with prioritization given to columns in this
722 order:
723 1. the provided named columns
724 2. spatial and temporal columns
725 3. the rest of the columns
727 The table is sorted in-place, and is also returned for convenience.
729 Parameters
730 ----------
731 table : `astropy.table.Table`
732 The table to sort
733 dimensions : `list` [``Dimension``]
734 The dimensions of the dataIds in the table (the dimensions should be
735 the same for all the dataIds). Used to determine if the column is
736 spatial, temporal, or neither.
737 sort_first : `list` [`str`]
738 The names of columns that should be sorted first, before spatial and
739 temporal columns.
741 Returns
742 -------
743 `astropy.table.Table`
744 For convenience, the table that has been sorted.
745 """
746 # For sorting we want to ignore the id
747 # We also want to move temporal or spatial dimensions earlier
748 sort_first = sort_first or []
749 sort_early = []
750 sort_late = []
751 for dim in dimensions:
752 if dim.spatial or dim.temporal:
753 sort_early.extend(dim.required.names)
754 else:
755 sort_late.append(str(dim))
756 sort_keys = sort_first + sort_early + sort_late
757 # The required names above means that we have the possibility of
758 # repeats of sort keys. Now have to remove them
759 # (order is retained by dict creation).
760 sort_keys = list(dict.fromkeys(sort_keys).keys())
762 table.sort(sort_keys)
763 return table