Coverage for python/lsst/daf/butler/cli/utils.py : 38%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import click
23import click.testing
24from contextlib import contextmanager
25import copy
26from functools import partial
27import io
28import os
29import textwrap
30import traceback
31from unittest.mock import MagicMock, patch
32import uuid
33import yaml
35from .cliLog import CliLog
36from ..core.utils import iterable
39# CLI_MOCK_ENV is set by some tests as an environment variable, it
40# indicates to the cli_handle_exception function that instead of executing the
41# command implementation function it should use the Mocker class for unit test
42# verification.
43mockEnvVarKey = "CLI_MOCK_ENV"
44mockEnvVar = {mockEnvVarKey: "1"}
46# This is used as the metavar argument to Options that accept multiple string
47# inputs, which may be comma-separarated. For example:
48# --my-opt foo,bar --my-opt baz.
49# Other arguments to the Option should include multiple=true and
50# callback=split_kv.
51typeStrAcceptsMultiple = "TEXT ..."
52typeStrAcceptsSingle = "TEXT"
54# For parameters that support key-value inputs, this defines the separator
55# for those inputs.
56split_kv_separator = "="
59def textTypeStr(multiple):
60 """Get the text type string for CLI help documentation.
62 Parameters
63 ----------
64 multiple : `bool`
65 True if multiple text values are allowed, False if only one value is
66 allowed.
68 Returns
69 -------
70 textTypeStr : `str`
71 The type string to use.
72 """
73 return typeStrAcceptsMultiple if multiple else typeStrAcceptsSingle
76class Mocker:
78 mock = MagicMock()
80 def __init__(self, *args, **kwargs):
81 """Mocker is a helper class for unit tests. It can be imported and
82 called and later imported again and call can be verified.
84 For convenience, constructor arguments are forwarded to the call
85 function.
86 """
87 self.__call__(*args, **kwargs)
89 def __call__(self, *args, **kwargs):
90 """Creates a MagicMock and stores it in a static variable that can
91 later be verified.
92 """
93 Mocker.mock(*args, **kwargs)
95 @classmethod
96 def reset(cls):
97 cls.mock.reset_mock()
100class LogCliRunner(click.testing.CliRunner):
101 """A test runner to use when the logging system will be initialized by code
102 under test, calls CliLog.resetLog(), which undoes any logging setup that
103 was done with the CliLog interface.
105 lsst.log modules can not be set back to an uninitialized state (python
106 logging modules can be set back to NOTSET), instead they are set to
107 `CliLog.defaultLsstLogLevel`."""
109 def invoke(self, *args, **kwargs):
110 result = super().invoke(*args, **kwargs)
111 CliLog.resetLog()
112 return result
115def clickResultMsg(result):
116 """Get a standard assert message from a click result
118 Parameters
119 ----------
120 result : click.Result
121 The result object returned from click.testing.CliRunner.invoke
123 Returns
124 -------
125 msg : `str`
126 The message string.
127 """
128 msg = f"""\noutput: {result.output}\nexception: {result.exception}"""
129 if result.exception:
130 msg += f"""\ntraceback: {"".join(traceback.format_tb(result.exception.__traceback__))}"""
131 return msg
134@contextmanager
135def command_test_env(runner, commandModule, commandName):
136 """A context manager that creates (and then cleans up) an environment that
137 provides a CLI plugin command with the given name.
139 Parameters
140 ----------
141 runner : click.testing.CliRunner
142 The test runner to use to create the isolated filesystem.
143 commandModule : `str`
144 The importable module that the command can be imported from.
145 commandName : `str`
146 The name of the command being published to import.
147 """
148 with runner.isolated_filesystem():
149 with open("resources.yaml", "w") as f:
150 f.write(yaml.dump({"cmd": {"import": commandModule, "commands": [commandName]}}))
151 # Add a colon to the end of the path on the next line, this tests the
152 # case where the lookup in LoaderCLI._getPluginList generates an empty
153 # string in one of the list entries and verifies that the empty string
154 # is properly stripped out.
155 with patch.dict("os.environ", {"DAF_BUTLER_PLUGINS": f"{os.path.realpath(f.name)}:"}):
156 yield
159def addArgumentHelp(doc, helpText):
160 """Add a Click argument's help message to a function's documentation.
162 This is needed because click presents arguments in the order the argument
163 decorators are applied to a function, top down. But, the evaluation of the
164 decorators happens bottom up, so if arguments just append their help to the
165 function's docstring, the argument descriptions appear in reverse order
166 from the order they are applied in.
168 Parameters
169 ----------
170 doc : `str`
171 The function's docstring.
172 helpText : `str`
173 The argument's help string to be inserted into the function's
174 docstring.
176 Returns
177 -------
178 doc : `str`
179 Updated function documentation.
180 """
181 if doc is None: 181 ↛ 182line 181 didn't jump to line 182, because the condition on line 181 was never true
182 doc = helpText
183 else:
184 doclines = doc.splitlines()
185 doclines.insert(1, helpText)
186 doclines.insert(1, "\n")
187 doc = "\n".join(doclines)
188 return doc
191def split_commas(context, param, values):
192 """Process a tuple of values, where each value may contain comma-separated
193 values, and return a single list of all the passed-in values.
195 This function can be passed to the 'callback' argument of a click.option to
196 allow it to process comma-separated values (e.g. "--my-opt a,b,c").
198 Parameters
199 ----------
200 context : `click.Context` or `None`
201 The current execution context. Unused, but Click always passes it to
202 callbacks.
203 param : `click.core.Option` or `None`
204 The parameter being handled. Unused, but Click always passes it to
205 callbacks.
206 values : [`str`]
207 All the values passed for this option. Strings may contain commas,
208 which will be treated as delimiters for separate values.
210 Returns
211 -------
212 list of string
213 The passed in values separated by commas and combined into a single
214 list.
215 """
216 if values is None:
217 return values
218 valueList = []
219 for value in iterable(values):
220 valueList.extend(value.split(","))
221 return tuple(valueList)
224def split_kv(context, param, values, choice=None, multiple=True, normalize=False, separator="=",
225 unseparated_okay=False, return_type=dict, default_key="", reverse_kv=False):
226 """Process a tuple of values that are key-value pairs separated by a given
227 separator. Multiple pairs may be comma separated. Return a dictionary of
228 all the passed-in values.
230 This function can be passed to the 'callback' argument of a click.option to
231 allow it to process comma-separated values (e.g. "--my-opt a=1,b=2").
233 Parameters
234 ----------
235 context : `click.Context` or `None`
236 The current execution context. Unused, but Click always passes it to
237 callbacks.
238 param : `click.core.Option` or `None`
239 The parameter being handled. Unused, but Click always passes it to
240 callbacks.
241 values : [`str`]
242 All the values passed for this option. Strings may contain commas,
243 which will be treated as delimiters for separate values.
244 choice : `click.Choice`, optional
245 If provided, verify each value is a valid choice using the provided
246 `click.Choice` instance. If None, no verification will be done. By
247 default None
248 multiple : `bool`, optional
249 If true, the value may contain multiple comma-separated values. By
250 default True.
251 normalize : `bool`, optional
252 If True and `choice.case_sensitive == False`, normalize the string the
253 user provided to match the choice's case. By default False.
254 separator : str, optional
255 The character that separates key-value pairs. May not be a comma or an
256 empty space (for space separators use Click's default implementation
257 for tuples; `type=(str, str)`). By default "=".
258 unseparated_okay : `bool`, optional
259 If True, allow values that do not have a separator. They will be
260 returned in the values dict as a tuple of values in the key '', that
261 is: `values[''] = (unseparated_values, )`. By default False.
262 return_type : `type`, must be `dict` or `tuple`
263 The type of the value that should be returned.
264 If `dict` then the returned object will be a dict, for each item in
265 values, the value to the left of the separator will be the key and the
266 value to the right of the separator will be the value.
267 If `tuple` then the returned object will be a tuple. Each item in the
268 tuple will be 2-item tuple, the first item will be the value to the
269 left of the separator and the second item will be the value to the
270 right. By default `dict`.
271 default_key : `Any`
272 The key to use if a value is passed that is not a key-value pair.
273 (Passing values that are not key-value pairs requires
274 ``unseparated_okay`` to be `True`.)
275 reverse_kv : bool
276 If true then for each item in values, the value to the left of the
277 separator is treated as the value and the value to the right of the
278 separator is treated as the key. By default False.
280 Returns
281 -------
282 values : `dict` [`str`, `str`]
283 The passed-in values in dict form.
285 Raises
286 ------
287 `click.ClickException`
288 Raised if the separator is not found in an entry, or if duplicate keys
289 are encountered.
290 """
292 def norm(val):
293 """If `normalize` is True and `choice` is not `None`, find the value
294 in the available choices and return the value as spelled in the
295 choices.
297 Assumes that val exists in choices; `split_kv` uses the `choice`
298 instance to verify val is a valid choice.
299 """
300 if normalize and choice is not None:
301 v = val.casefold()
302 for opt in choice.choices:
303 if opt.casefold() == v:
304 return opt
305 return val
307 class RetDict:
309 def __init__(self):
310 self.ret = {}
312 def add(self, key, val):
313 if reverse_kv:
314 key, val = val, key
315 if key in self.ret:
316 raise click.ClickException(f"Duplicate entries for '{k}' in '{values}'")
317 self.ret[key] = val
319 def get(self):
320 return self.ret
322 class RetTuple:
324 def __init__(self):
325 self.ret = []
327 def add(self, key, val):
328 if reverse_kv:
329 key, val = val, key
330 self.ret.append((key, val))
332 def get(self):
333 return tuple(self.ret)
335 if separator in (",", " "):
336 raise RuntimeError(f"'{separator}' is not a supported separator for key-value pairs.")
337 vals = values # preserve the original argument for error reporting below.
338 if return_type is dict:
339 ret = RetDict()
340 elif return_type is tuple:
341 ret = RetTuple()
342 else:
343 raise click.ClickException(f"Internal error: invalid return type '{return_type}' for split_kv.")
344 if multiple:
345 vals = split_commas(context, param, vals)
346 for val in iterable(vals):
347 if unseparated_okay and separator not in val:
348 if choice is not None:
349 choice(val) # will raise if val is an invalid choice
350 ret.add(default_key, norm(val))
351 else:
352 try:
353 k, v = val.split(separator)
354 if choice is not None:
355 choice(v) # will raise if val is an invalid choice
356 except ValueError:
357 raise click.ClickException(
358 f"Could not parse key-value pair '{val}' using separator '{separator}', "
359 f"with multiple values {'allowed' if multiple else 'not allowed'}.")
360 ret.add(k, norm(v))
361 return ret.get()
364def to_upper(context, param, value):
365 """Convert a value to upper case.
367 Parameters
368 ----------
369 context : click.Context
371 values : string
372 The value to be converted.
374 Returns
375 -------
376 string
377 A copy of the passed-in value, converted to upper case.
378 """
379 return value.upper()
382def unwrap(val):
383 """Remove newlines and leading whitespace from a multi-line string with
384 a consistent indentation level.
386 The first line of the string may be only a newline or may contain text
387 followed by a newline, either is ok. After the first line, each line must
388 begin with a consistant amount of whitespace. So, content of a
389 triple-quoted string may begin immediately after the quotes, or the string
390 may start with a newline. Each line after that must be the same amount of
391 indentation/whitespace followed by text and a newline. The last line may
392 end with a new line but is not required to do so.
394 Parameters
395 ----------
396 val : `str`
397 The string to change.
399 Returns
400 -------
401 strippedString : `str`
402 The string with newlines, indentation, and leading and trailing
403 whitespace removed.
404 """
405 def splitSection(val):
406 if not val.startswith("\n"): 406 ↛ 410line 406 didn't jump to line 410, because the condition on line 406 was never false
407 firstLine, _, val = val.partition("\n")
408 firstLine += " "
409 else:
410 firstLine = ""
411 return (firstLine + textwrap.dedent(val).replace("\n", " ")).strip()
413 return "\n\n".join([splitSection(s) for s in val.split("\n\n")])
416def cli_handle_exception(func, *args, **kwargs):
417 """Wrap a function call in an exception handler that raises a
418 ClickException if there is an Exception.
420 Also provides support for unit testing by testing for an environment
421 variable, and if it is present prints the function name, args, and kwargs
422 to stdout so they can be read and verified by the unit test code.
424 Parameters
425 ----------
426 func : function
427 A function to be called and exceptions handled. Will pass args & kwargs
428 to the function.
430 Returns
431 -------
432 The result of calling func.
434 Raises
435 ------
436 click.ClickException
437 An exception to be handled by the Click CLI tool.
438 """
439 if mockEnvVarKey in os.environ:
440 Mocker(*args, **kwargs)
441 return
442 try:
443 return func(*args, **kwargs)
444 except Exception:
445 msg = io.StringIO()
446 msg.write("An error occurred during command execution:\n")
447 traceback.print_exc(file=msg)
448 raise click.ClickException(msg.getvalue())
451class option_section: # noqa: N801
452 """Decorator to add a section label between options in the help text of a
453 command.
455 Parameters
456 ----------
457 sectionText : `str`
458 The text to print in the section identifier.
459 """
461 def __init__(self, sectionText):
462 self.sectionText = "\n" + sectionText
464 def __call__(self, f):
465 # Generate a parameter declaration that will be unique for this
466 # section.
467 return click.option(f"--option-section-{str(uuid.uuid4())}",
468 sectionText=self.sectionText,
469 cls=OptionSection)(f)
472class MWPath(click.Path):
473 """Overrides click.Path to implement file-does-not-exist checking.
475 Changes the definition of ``exists` so that `True` indicates the location
476 (file or directory) must exist, `False` indicates the location must *not*
477 exist, and `None` indicates that the file may exist or not. The standard
478 definition for the `click.Path` ``exists`` parameter is that for `True` a
479 location must exist, but `False` means it is not required to exist (not
480 that it is required to not exist).
482 Parameters
483 ----------
484 exists : `True`, `False`, or `None`
485 If `True`, the location (file or directory) indicated by the caller
486 must exist. If `False` the location must not exist. If `None`, the
487 location may exist or not.
489 For other parameters see `click.Path`.
490 """
492 def __init__(self, exists=None, file_okay=True, dir_okay=True,
493 writable=False, readable=True, resolve_path=False,
494 allow_dash=False, path_type=None):
495 self.mustNotExist = exists is False
496 if exists is None:
497 exists = False
498 super().__init__(exists, file_okay, dir_okay, writable, readable,
499 resolve_path, allow_dash, path_type)
501 def convert(self, value, param, ctx):
502 """Called by click.ParamType to "convert values through types".
503 `click.Path` uses this step to verify Path conditions."""
504 if self.mustNotExist and os.path.exists(value):
505 self.fail(f'{self.path_type} "{value}" should not exist.')
506 return super().convert(value, param, ctx)
509class MWOption(click.Option):
510 """Overrides click.Option with desired behaviors."""
512 def make_metavar(self):
513 """Overrides `click.Option.make_metavar`. Makes the metavar for the
514 help menu. Adds a space and an elipsis after the metavar name if
515 the option accepts multiple inputs, otherwise defers to the base
516 implementation.
518 By default click does not add an elipsis when multiple is True and
519 nargs is 1. And when nargs does not equal 1 click adds an elipsis
520 without a space between the metavar and the elipsis, but we prefer a
521 space between.
523 Does not get called for some option types (e.g. flag) so metavar
524 transformation that must apply to all types should be applied in
525 get_help_record.
526 """
527 metavar = super().make_metavar()
528 if self.multiple and self.nargs == 1:
529 metavar += " ..."
530 elif self.nargs != 1:
531 metavar = f"{metavar[:-3]} ..."
532 return metavar
535class MWArgument(click.Argument):
536 """Overrides click.Argument with desired behaviors."""
538 def make_metavar(self):
539 """Overrides `click.Option.make_metavar`. Makes the metavar for the
540 help menu. Always adds a space and an elipsis (' ...') after the
541 metavar name if the option accepts multiple inputs.
543 By default click adds an elipsis without a space between the metavar
544 and the elipsis, but we prefer a space between.
546 Returns
547 -------
548 metavar : `str`
549 The metavar value.
550 """
551 metavar = super().make_metavar()
552 if self.nargs != 1:
553 metavar = f"{metavar[:-3]} ..."
554 return metavar
557class OptionSection(MWOption):
558 """Implements an Option that prints a section label in the help text and
559 does not pass any value to the command function.
561 This class does a bit of hackery to add a section label to a click command
562 help output: first, `expose_value` is set to `False` so that no value is
563 passed to the command function. Second, this class overrides
564 `click.Option.get_help_record` to return the section label string without
565 any prefix so that it stands out as a section label.
567 The intention for this implementation is to do minimally invasive overrides
568 of the click classes so as to be robust and easy to fix if the click
569 internals change.
571 Parameters
572 ----------
573 sectionName : `str`
574 The parameter declaration for this option. It is not shown to the user,
575 it must be unique within the command. If using the `section` decorator
576 to add a section to a command's options, the section name is
577 auto-generated.
578 sectionText : `str`
579 The text to print in the section identifier.
580 """
582 def __init__(self, sectionName, sectionText):
583 super().__init__(sectionName, expose_value=False)
584 self.sectionText = sectionText
586 def get_help_record(self, ctx):
587 return (self.sectionText, "")
590class MWOptionDecorator:
591 """Wraps the click.option decorator to enable shared options to be declared
592 and allows inspection of the shared option.
593 """
595 def __init__(self, *param_decls, **kwargs):
596 self.partialOpt = partial(click.option, *param_decls, cls=partial(MWOption),
597 **kwargs)
598 opt = click.Option(param_decls, **kwargs)
599 self._name = opt.name
600 self._opts = opt.opts
602 def name(self):
603 """Get the name that will be passed to the command function for this
604 option."""
605 return self._name
607 def opts(self):
608 """Get the flags that will be used for this option on the command
609 line."""
610 return self._opts
612 def __call__(self, *args, **kwargs):
613 return self.partialOpt(*args, **kwargs)
616class MWArgumentDecorator:
617 """Wraps the click.argument decorator to enable shared arguments to be
618 declared. """
620 def __init__(self, *param_decls, **kwargs):
621 self._helpText = kwargs.pop("help", None)
622 self.partialArg = partial(click.argument, *param_decls, cls=MWArgument, **kwargs)
624 def __call__(self, *args, help=None, **kwargs):
625 def decorator(f):
626 if help is not None:
627 self._helpText = help
628 if self._helpText: 628 ↛ 630line 628 didn't jump to line 630, because the condition on line 628 was never false
629 f.__doc__ = addArgumentHelp(f.__doc__, self._helpText)
630 return self.partialArg(*args, **kwargs)(f)
631 return decorator
634class MWCommand(click.Command):
635 """Command subclass that stores a copy of the args list for use by the
636 command."""
638 def parse_args(self, ctx, args):
639 MWCtxObj.getFrom(ctx).args = copy.copy(args)
640 super().parse_args(ctx, args)
643class MWCtxObj():
644 """Helper object for managing the `click.Context.obj` parameter, allows
645 obj data to be managed in a consistent way.
647 `Context.obj` defaults to None. `MWCtxObj.getFrom(ctx)` can be used to
648 initialize the obj if needed and return a new or existing MWCtxObj.
650 Attributes
651 ----------
652 args : `list` [`str`]
653 The list of arguments (argument values, option flags, and option
654 values), split using whitespace, that were passed in on the command
655 line for the subcommand represented by the parent context object.
656 """
658 def __init__(self):
660 self.args = None
662 @staticmethod
663 def getFrom(ctx):
664 """If needed, initialize `ctx.obj` with a new MWCtxObj, and return the
665 new or already existing MWCtxObj."""
666 if ctx.obj is not None:
667 return ctx.obj
668 ctx.obj = MWCtxObj()
669 return ctx.obj