Coverage for python/lsst/daf/butler/cli/utils.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23import click.testing
24from contextlib import contextmanager
25import copy
26from functools import partial
27import io
28import os
29import textwrap
30import traceback
31from unittest.mock import MagicMock, patch
32import uuid
33import yaml
35from .cliLog import CliLog
36from ..core.utils import iterable
39# CLI_MOCK_ENV is set by some tests as an environment variable, it
40# indicates to the cli_handle_exception function that instead of executing the
41# command implementation function it should use the Mocker class for unit test
42# verification.
43mockEnvVarKey = "CLI_MOCK_ENV"
44mockEnvVar = {mockEnvVarKey: "1"}
46# This is used as the metavar argument to Options that accept multiple string
47# inputs, which may be comma-separarated. For example:
48# --my-opt foo,bar --my-opt baz.
49# Other arguments to the Option should include multiple=true and
50# callback=split_kv.
51typeStrAcceptsMultiple = "TEXT ..."
52typeStrAcceptsSingle = "TEXT"
55def textTypeStr(multiple):
56 """Get the text type string for CLI help documentation.
58 Parameters
59 ----------
60 multiple : `bool`
61 True if multiple text values are allowed, False if only one value is
62 allowed.
64 Returns
65 -------
66 textTypeStr : `str`
67 The type string to use.
68 """
69 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
72# For parameters that support key-value inputs, this defines the separator
73# for those inputs.
74split_kv_separator = "="
77class Mocker:
79 mock = MagicMock()
81 def __init__(self, *args, **kwargs):
82 """Mocker is a helper class for unit tests. It can be imported and
83 called and later imported again and call can be verified.
85 For convenience, constructor arguments are forwarded to the call
86 function.
87 """
88 self.__call__(*args, **kwargs)
90 def __call__(self, *args, **kwargs):
91 """Creates a MagicMock and stores it in a static variable that can
92 later be verified.
93 """
94 Mocker.mock(*args, **kwargs)
96 @classmethod
97 def reset(cls):
98 cls.mock.reset_mock()
101class LogCliRunner(click.testing.CliRunner):
102 """A test runner to use when the logging system will be initialized by code
103 under test, calls CliLog.resetLog(), which undoes any logging setup that
104 was done with the CliLog interface.
106 lsst.log modules can not be set back to an uninitialized state (python
107 logging modules can be set back to NOTSET), instead they are set to
108 `CliLog.defaultLsstLogLevel`."""
110 def invoke(self, *args, **kwargs):
111 result = super().invoke(*args, **kwargs)
112 CliLog.resetLog()
113 return result
116def clickResultMsg(result):
117 """Get a standard assert message from a click result
119 Parameters
120 ----------
121 result : click.Result
122 The result object returned from click.testing.CliRunner.invoke
124 Returns
125 -------
126 msg : `str`
127 The message string.
128 """
129 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
130 if result.exception:
131 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
132 return msg
135@contextmanager
136def command_test_env(runner, commandModule, commandName):
137 """A context manager that creates (and then cleans up) an environment that
138 provides a CLI plugin command with the given name.
140 Parameters
141 ----------
142 runner : click.testing.CliRunner
143 The test runner to use to create the isolated filesystem.
144 commandModule : `str`
145 The importable module that the command can be imported from.
146 commandName : `str`
147 The name of the command being published to import.
148 """
149 with runner.isolated_filesystem():
150 with open("resources.yaml", "w") as f:
151 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
152 # Add a colon to the end of the path on the next line, this tests the
153 # case where the lookup in LoaderCLI._getPluginList generates an empty
154 # string in one of the list entries and verifies that the empty string
155 # is properly stripped out.
156 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
157 yield
160def addArgumentHelp(doc, helpText):
161 """Add a Click argument's help message to a function's documentation.
163 This is needed because click presents arguments in the order the argument
164 decorators are applied to a function, top down. But, the evaluation of the
165 decorators happens bottom up, so if arguments just append their help to the
166 function's docstring, the argument descriptions appear in reverse order
167 from the order they are applied in.
169 Parameters
170 ----------
171 doc : `str`
172 The function's docstring.
173 helpText : `str`
174 The argument's help string to be inserted into the function's
175 docstring.
177 Returns
178 -------
179 doc : `str`
180 Updated function documentation.
181 """
182 if doc is None: 182 ↛ 183line 182 didn't jump to line 183, because the condition on line 182 was never true
183 doc = helpText
184 else:
185 doclines = doc.splitlines()
186 doclines.insert(1, helpText)
187 doclines.insert(1, "\n")
188 doc = "\n".join(doclines)
189 return doc
192def split_commas(context, param, values):
193 """Process a tuple of values, where each value may contain comma-separated
194 values, and return a single list of all the passed-in values.
196 This function can be passed to the 'callback' argument of a click.option to
197 allow it to process comma-separated values (e.g. "--my-opt a,b,c").
199 Parameters
200 ----------
201 context : `click.Context` or `None`
202 The current execution context. Unused, but Click always passes it to
203 callbacks.
204 param : `click.core.Option` or `None`
205 The parameter being handled. Unused, but Click always passes it to
206 callbacks.
207 values : [`str`]
208 All the values passed for this option. Strings may contain commas,
209 which will be treated as delimiters for separate values.
211 Returns
212 -------
213 list of string
214 The passed in values separated by commas and combined into a single
215 list.
216 """
217 if values is None:
218 return values
219 valueList = []
220 for value in iterable(values):
221 valueList.extend(value.split(","))
222 return tuple(valueList)
225def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=",
226 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False):
227 """Process a tuple of values that are key-value pairs separated by a given
228 separator. Multiple pairs may be comma separated. Return a dictionary of
229 all the passed-in values.
231 This function can be passed to the 'callback' argument of a click.option to
232 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
234 Parameters
235 ----------
236 context : `click.Context` or `None`
237 The current execution context. Unused, but Click always passes it to
238 callbacks.
239 param : `click.core.Option` or `None`
240 The parameter being handled. Unused, but Click always passes it to
241 callbacks.
242 values : [`str`]
243 All the values passed for this option. Strings may contain commas,
244 which will be treated as delimiters for separate values.
245 choice : `click.Choice`, optional
246 If provided, verify each value is a valid choice using the provided
247 `click.Choice` instance. If None, no verification will be done. By
248 default None
249 multiple : `bool`, optional
250 If true, the value may contain multiple comma-separated values. By
251 default True.
252 normalize : `bool`, optional
253 If True and `choice.case_sensitive == False`, normalize the string the
254 user provided to match the choice's case. By default False.
255 separator : str, optional
256 The character that separates key-value pairs. May not be a comma or an
257 empty space (for space separators use Click's default implementation
258 for tuples; `type=(str, str)`). By default "=".
259 unseparated_okay : `bool`, optional
260 If True, allow values that do not have a separator. They will be
261 returned in the values dict as a tuple of values in the key '', that
262 is: `values[''] = (unseparated_values, )`. By default False.
263 return_type : `type`, must be `dict` or `tuple`
264 The type of the value that should be returned.
265 If `dict` then the returned object will be a dict, for each item in
266 values, the value to the left of the separator will be the key and the
267 value to the right of the separator will be the value.
268 If `tuple` then the returned object will be a tuple. Each item in the
269 tuple will be 2-item tuple, the first item will be the value to the
270 left of the separator and the second item will be the value to the
271 right. By default `dict`.
272 default_key : `Any`
273 The key to use if a value is passed that is not a key-value pair.
274 (Passing values that are not key-value pairs requires
275 ``unseparated_okay`` to be `True`.)
276 reverse_kv : bool
277 If true then for each item in values, the value to the left of the
278 separator is treated as the value and the value to the right of the
279 separator is treated as the key. By default False.
281 Returns
282 -------
283 values : `dict` [`str`, `str`]
284 The passed-in values in dict form.
286 Raises
287 ------
288 `click.ClickException`
289 Raised if the separator is not found in an entry, or if duplicate keys
290 are encountered.
291 """
293 def norm(val):
294 """If `normalize` is True and `choice` is not `None`, find the value
295 in the available choices and return the value as spelled in the
296 choices.
298 Assumes that val exists in choices; `split_kv` uses the `choice`
299 instance to verify val is a valid choice.
300 """
301 if normalize and choice is not None:
302 v = val.casefold()
303 for opt in choice.choices:
304 if opt.casefold() == v:
305 return opt
306 return val
308 class RetDict:
310 def __init__(self):
311 self.ret = {}
313 def add(self, key, val):
314 if reverse_kv:
315 key, val = val, key
316 if key in self.ret:
317 raise click.ClickException(f"Duplicate entries for '{k}' in '{values}'")
318 self.ret[key] = val
320 def get(self):
321 return self.ret
323 class RetTuple:
325 def __init__(self):
326 self.ret = []
328 def add(self, key, val):
329 if reverse_kv:
330 key, val = val, key
331 self.ret.append((key, val))
333 def get(self):
334 return tuple(self.ret)
336 if separator in (",", " "):
337 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
338 vals = values # preserve the original argument for error reporting below.
339 if return_type is dict:
340 ret = RetDict()
341 elif return_type is tuple:
342 ret = RetTuple()
343 else:
344 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.")
345 if multiple:
346 vals = split_commas(context, param, vals)
347 for val in iterable(vals):
348 if unseparated_okay and separator not in val:
349 if choice is not None:
350 choice(val) # will raise if val is an invalid choice
351 ret.add(default_key, norm(val))
352 else:
353 try:
354 k, v = val.split(separator)
355 if choice is not None:
356 choice(v) # will raise if val is an invalid choice
357 except ValueError:
358 raise click.ClickException(
359 f"Could not parse key-value pair '{val}' using separator '{separator}', "
360 f"with multiple values {'allowed' if multiple else 'not allowed'}.")
361 ret.add(k, norm(v))
362 return ret.get()
365def to_upper(context, param, value):
366 """Convert a value to upper case.
368 Parameters
369 ----------
370 context : click.Context
372 values : string
373 The value to be converted.
375 Returns
376 -------
377 string
378 A copy of the passed-in value, converted to upper case.
379 """
380 return value.upper()
383def unwrap(val):
384 """Remove newlines and leading whitespace from a multi-line string with
385 a consistent indentation level.
387 The first line of the string may be only a newline or may contain text
388 followed by a newline, either is ok. After the first line, each line must
389 begin with a consistant amount of whitespace. So, content of a
390 triple-quoted string may begin immediately after the quotes, or the string
391 may start with a newline. Each line after that must be the same amount of
392 indentation/whitespace followed by text and a newline. The last line may
393 end with a new line but is not required to do so.
395 Parameters
396 ----------
397 val : `str`
398 The string to change.
400 Returns
401 -------
402 strippedString : `str`
403 The string with newlines, indentation, and leading and trailing
404 whitespace removed.
405 """
406 def splitSection(val):
407 if not val.startswith("\n"): 407 ↛ 411line 407 didn't jump to line 411, because the condition on line 407 was never false
408 firstLine, _, val = val.partition("\n")
409 firstLine += " "
410 else:
411 firstLine = ""
412 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
414 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
417def cli_handle_exception(func, *args, **kwargs):
418 """Wrap a function call in an exception handler that raises a
419 ClickException if there is an Exception.
421 Also provides support for unit testing by testing for an environment
422 variable, and if it is present prints the function name, args, and kwargs
423 to stdout so they can be read and verified by the unit test code.
425 Parameters
426 ----------
427 func : function
428 A function to be called and exceptions handled. Will pass args & kwargs
429 to the function.
431 Returns
432 -------
433 The result of calling func.
435 Raises
436 ------
437 click.ClickException
438 An exception to be handled by the Click CLI tool.
439 """
440 if mockEnvVarKey in os.environ:
441 Mocker(*args, **kwargs)
442 return
443 try:
444 return func(*args, **kwargs)
445 except Exception:
446 msg = io.StringIO()
447 msg.write("An error occurred during command execution:\n")
448 traceback.print_exc(file=msg)
449 raise click.ClickException(msg.getvalue())
452class option_section: # noqa: N801
453 """Decorator to add a section label between options in the help text of a
454 command.
456 Parameters
457 ----------
458 sectionText : `str`
459 The text to print in the section identifier.
460 """
462 def __init__(self, sectionText):
463 self.sectionText = "\n" + sectionText
465 def __call__(self, f):
466 # Generate a parameter declaration that will be unique for this
467 # section.
468 return click.option(f"--option-section-{str(uuid.uuid4())}",
469 sectionText=self.sectionText,
470 cls=OptionSection)(f)
473class MWPath(click.Path):
474 """Overrides click.Path to implement file-does-not-exist checking.
476 Changes the definition of ``exists` so that `True` indicates the location
477 (file or directory) must exist, `False` indicates the location must *not*
478 exist, and `None` indicates that the file may exist or not. The standard
479 definition for the `click.Path` ``exists`` parameter is that for `True` a
480 location must exist, but `False` means it is not required to exist (not
481 that it is required to not exist).
483 Parameters
484 ----------
485 exists : `True`, `False`, or `None`
486 If `True`, the location (file or directory) indicated by the caller
487 must exist. If `False` the location must not exist. If `None`, the
488 location may exist or not.
490 For other parameters see `click.Path`.
491 """
493 def __init__(self, exists=None, file_okay=True, dir_okay=True,
494 writable=False, readable=True, resolve_path=False,
495 allow_dash=False, path_type=None):
496 self.mustNotExist = exists is False
497 if exists is None:
498 exists = False
499 super().__init__(exists, file_okay, dir_okay, writable, readable,
500 resolve_path, allow_dash, path_type)
502 def convert(self, value, param, ctx):
503 """Called by click.ParamType to "convert values through types".
504 `click.Path` uses this step to verify Path conditions."""
505 if self.mustNotExist and os.path.exists(value):
506 self.fail(f'{self.path_type} "{value}" should not exist.')
507 return super().convert(value, param, ctx)
510class MWOption(click.Option):
511 """Overrides click.Option with desired behaviors."""
513 def make_metavar(self):
514 """Overrides `click.Option.make_metavar`. Makes the metavar for the
515 help menu. Adds a space and an elipsis after the metavar name if
516 the option accepts multiple inputs, otherwise defers to the base
517 implementation.
519 By default click does not add an elipsis when multiple is True and
520 nargs is 1. And when nargs does not equal 1 click adds an elipsis
521 without a space between the metavar and the elipsis, but we prefer a
522 space between.
524 Does not get called for some option types (e.g. flag) so metavar
525 transformation that must apply to all types should be applied in
526 get_help_record.
527 """
528 metavar = super().make_metavar()
529 if self.multiple and self.nargs == 1:
530 metavar += " ..."
531 elif self.nargs != 1:
532 metavar = f"{metavar[:-3]} ..."
533 return metavar
536class MWArgument(click.Argument):
537 """Overrides click.Argument with desired behaviors."""
539 def make_metavar(self):
540 """Overrides `click.Option.make_metavar`. Makes the metavar for the
541 help menu. Always adds a space and an elipsis (' ...') after the
542 metavar name if the option accepts multiple inputs.
544 By default click adds an elipsis without a space between the metavar
545 and the elipsis, but we prefer a space between.
547 Returns
548 -------
549 metavar : `str`
550 The metavar value.
551 """
552 metavar = super().make_metavar()
553 if self.nargs != 1:
554 metavar = f"{metavar[:-3]} ..."
555 return metavar
558class OptionSection(MWOption):
559 """Implements an Option that prints a section label in the help text and
560 does not pass any value to the command function.
562 This class does a bit of hackery to add a section label to a click command
563 help output: first, `expose_value` is set to `False` so that no value is
564 passed to the command function. Second, this class overrides
565 `click.Option.get_help_record` to return the section label string without
566 any prefix so that it stands out as a section label.
568 The intention for this implementation is to do minimally invasive overrides
569 of the click classes so as to be robust and easy to fix if the click
570 internals change.
572 Parameters
573 ----------
574 sectionName : `str`
575 The parameter declaration for this option. It is not shown to the user,
576 it must be unique within the command. If using the `section` decorator
577 to add a section to a command's options, the section name is
578 auto-generated.
579 sectionText : `str`
580 The text to print in the section identifier.
581 """
583 def __init__(self, sectionName, sectionText):
584 super().__init__(sectionName, expose_value=False)
585 self.sectionText = sectionText
587 def get_help_record(self, ctx):
588 return (self.sectionText, "")
591class MWOptionDecorator:
592 """Wraps the click.option decorator to enable shared options to be declared
593 and allows inspection of the shared option.
594 """
596 def __init__(self, *param_decls, **kwargs):
597 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption),
598 **kwargs)
599 opt = click.Option(param_decls, **kwargs)
600 self._name = opt.name
601 self._opts = opt.opts
603 def name(self):
604 """Get the name that will be passed to the command function for this
605 option."""
606 return self._name
608 def opts(self):
609 """Get the flags that will be used for this option on the command
610 line."""
611 return self._opts
613 def __call__(self, *args, **kwargs):
614 return self.partialOpt(*args, **kwargs)
617class MWArgumentDecorator:
618 """Wraps the click.argument decorator to enable shared arguments to be
619 declared. """
621 def __init__(self, *param_decls, **kwargs):
622 self._helpText = kwargs.pop("help", None)
623 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
625 def __call__(self, *args, help=None, **kwargs):
626 def decorator(f):
627 if help is not None:
628 self._helpText = help
629 if self._helpText: 629 ↛ 631line 629 didn't jump to line 631, because the condition on line 629 was never false
630 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
631 return self.partialArg(*args, **kwargs)(f)
632 return decorator
635class MWCommand(click.Command):
636 """Command subclass that stores a copy of the args list for use by the
637 command."""
639 def parse_args(self, ctx, args):
640 MWCtxObj.getFrom(ctx).args = copy.copy(args)
641 super().parse_args(ctx, args)
644class MWCtxObj():
645 """Helper object for managing the `click.Context.obj` parameter, allows
646 obj data to be managed in a consistent way.
648 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
649 initialize the obj if needed and return a new or existing MWCtxObj.
651 Attributes
652 ----------
653 args : `list` [`str`]
654 The list of arguments (argument values, option flags, and option
655 values), split using whitespace, that were passed in on the command
656 line for the subcommand represented by the parent context object.
657 """
659 def __init__(self):
661 self.args = None
663 @staticmethod
664 def getFrom(ctx):
665 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the
666 new or already existing MWCtxObj."""
667 if ctx.obj is not None:
668 return ctx.obj
669 ctx.obj = MWCtxObj()
670 return ctx.obj