Coverage for python/lsst/daf/butler/tests/cliOptionTestBase.py : 27%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of daf_butler.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22import abc
23import click
24import copy
25import inspect
26import os
27from unittest.mock import MagicMock
29from ..cli.utils import clickResultMsg, LogCliRunner, ParameterType, split_kv_separator
30from ..core.utils import iterable
33class MockCliTestHelper:
34 """Contains objects associated with a CLI test that calls a mock.
35 """
36 def __init__(self, cli=None, mock=None, expectedArgs=None, expectedKwargs=None):
37 self.cli = cli
38 self.mock = mock
39 self.expectedArgs = [] if expectedArgs is None else expectedArgs
40 self.expectedKwargs = {} if expectedKwargs is None else expectedKwargs
43class CliFactory:
45 @staticmethod
46 def noOp(optTestBase, parameterKwargs=None, cmdInitKwArgs=None):
47 """Produces a no-op cli function that supports the option class being
48 tested, and initializes the option class with the expected and
49 passed-in keyword arguments.
51 Uses `optTestBase.isArgument` and `optTestBase.isParameter` to
52 determine if the option should be initialzied with a `parameterType`
53 keyword argument, and sets it accordingly (`ParameterType.OPTION` is
54 the standard default for parameters so this only sets it for
55 `ARGUMENT`).
57 Parameters
58 ----------
59 optTestBase : `OptTestBase` subclass
60 The test being executed.
61 parameterKwargs : `dict` [`str`, `Any`], optional
62 A list of keyword arguments to pass to the parameter (Argument or
63 Option) constructor.
64 cmdInitKwArgs : `dict` [`str`, `Any`], optional
65 A list of keyword arguments to pass to the command constructor.
67 Returns
68 -------
69 cli : a click.Command.
70 A click command that can be invoked by the click test invoker.
71 """
72 cliArgs = copy.copy(parameterKwargs) if parameterKwargs is not None else {}
73 if "parameterType" not in cliArgs and optTestBase.isArgument and optTestBase.isParameter:
74 cliArgs["parameterType"] = ParameterType.ARGUMENT
76 if cmdInitKwArgs is None:
77 cmdInitKwArgs = {}
79 @click.command(**cmdInitKwArgs)
80 @optTestBase.optionClass(**cliArgs)
81 def cli(*args, **kwargs):
82 pass
83 return cli
85 @staticmethod
86 def mocked(optTestBase, expectedArgs=None, expectedKwargs=None, parameterKwargs=None):
87 """Produces a helper object with a cli function that supports the
88 option class being tested, the mock it will call, and args & kwargs
89 that the mock is expected to be called with. Initializes the option
90 class with the expected and passed-in keyword arguments.
92 Uses `optTestBase.isArgument` and `optTestBase.isParameter` to
93 determine if the option should be initialzied with a `parameterType`
94 keyword argument, and sets it accordingly (`ParameterType.OPTION` is
95 the standard default for parameters so this only sets it for
96 `ARGUMENT`).
98 Parameters
99 ----------
100 optTestBase : `OptTestBase` subclass
101 The test being executed.
102 expectedArgs : `list [`Any`], optional
103 A list of arguments the mock is expected to be called with, by
104 default None.
105 expectedKwargs : `dict` [`str`, `Any`], optional
106 A list of keyword arguments the mock is expected to be called with,
107 by default None.
108 parameterKwargs : `dict` [`str`, `Any`], optional
109 A list of keyword arguments to pass to the parameter (Argument or
110 Option) constructor.
112 Returns
113 -------
114 helper : `MockCliTestHelper`
115 The helper object.
116 """
117 cliArgs = copy.copy(parameterKwargs) if parameterKwargs is not None else {}
118 if "parameterType" not in cliArgs and optTestBase.isArgument and optTestBase.isParameter:
119 cliArgs["parameterType"] = ParameterType.ARGUMENT
121 helper = MockCliTestHelper(mock=MagicMock(),
122 expectedArgs=expectedArgs,
123 expectedKwargs=expectedKwargs)
125 @click.command()
126 @optTestBase.optionClass(**cliArgs)
127 def cli(*args, **kwargs):
128 helper.mock(*args, **kwargs)
129 helper.cli = cli
130 return helper
133class OptTestBase(abc.ABC):
134 """A test case base that is used with Opt...Test mixin classes to test
135 supported click option behaviors.
136 """
138 def setUp(self):
139 self.runner = LogCliRunner()
141 @property
142 def valueType(self):
143 """The value `type` of the click.Option."""
144 return str
146 @property
147 @abc.abstractmethod
148 def optionClass(self):
149 """The option class being tested"""
150 pass
152 @property
153 def optionKey(self):
154 """The option name as it appears as a function argument and in
155 subsequent uses (e.g. kwarg dicts); dashes are replaced by underscores.
156 """
157 return self.optionName.replace("-", "_")
159 @property
160 def optionFlag(self):
161 """The flag that is used on the command line for the option."""
162 return f"--{self.optionName}"
164 @property
165 def shortOptionFlag(self):
166 """The abbreviated flag that is used on the command line for the
167 option.
168 """
169 return f"-{self.shortOptionName}" if self.shortOptionName else None
171 @property
172 @abc.abstractmethod
173 def optionName(self):
174 """The option name, matches the option flag that appears on the command
175 line."""
176 pass
178 @property
179 def shortOptionName(self):
180 """The short option flag that can be used on the command line.
182 Returns
183 -------
184 shortOptionName : `str` or `None`
185 The short option, or None if a short option is not used for this
186 option.
187 """
188 return None
190 @property
191 def optionValue(self):
192 """The value to pass for the option flag when calling the test
193 command. If the option class restricts option values, by default
194 returns the first item from `self.optionClass.choices`, otherwise
195 returns a nonsense string.
196 """
197 if self.isChoice:
198 return self.choices[0]
199 return "foobarbaz"
201 @property
202 def optionMultipleValues(self):
203 """The value(s) to pass for the option flag when calling a test command
204 with multiple inputs.
206 Returns
207 -------
208 values : `list` [`str`]
209 A list of values, each item in the list will be passed to the
210 command with an option flag. Items in the list may be
211 comma-separated, e.g. ["foo", "bar,baz"].
212 """
213 # This return value matches the value returned by
214 # expectedMultipleValues.
215 return ("foo", "bar,baz")
217 @property
218 def optionMultipleKeyValues(self):
219 """The values to pass for the option flag when calling a test command
220 with multiple key-value inputs."""
221 return ["one=two,three=four", "five=six"]
223 @property
224 def expectedVal(self):
225 """The expected value to receive in the command function. Typically
226 that value is printed to stdout and compared with this value. By
227 default returns the same value as `self.optionValue`."""
228 if self.isChoice:
229 return self.expectedChoiceValues[0]
230 return self.optionValue
232 @property
233 def expectedValDefault(self):
234 """When the option is not required and not passed to the test command,
235 this is the expected default value to appear in the command function.
236 """
237 return None
239 @property
240 def expectedMultipleValues(self):
241 """The expected values to receive in the command function when a test
242 command is called with multiple inputs.
244 Returns
245 -------
246 expectedValues : `list` [`str`]
247 A list of expected values, e.g. ["foo", "bar", "baz"].
248 """
249 # This return value matches the value returned by optionMultipleValues.
250 return ("foo", "bar", "baz")
252 @property
253 def expectedMultipleKeyValues(self):
254 """The expected valuse to receive in the command function when a test
255 command is called with multiple key - value inputs. """
256 # These return values matches the values returned by
257 # optionMultipleKeyValues
258 return dict(one="two", three="four", five="six")
260 @property
261 def choices(self):
262 """Return the list of valid choices for the option."""
263 return self.optionClass.choices
265 @property
266 def expectedChoiceValues(self):
267 """Return the list of expected values for the option choices. Must
268 match the size and order of the list returned by `choices`."""
269 return self.choices
271 @property
272 def metavar(self):
273 """Return the metavar expected to be printed in help text after the
274 option flag(s). If `None`, won't run a test for the metavar value."""
275 return None
277 @property
278 def isArgument(self):
279 """True if the Parameter under test is an Argument, False if it is an
280 Option."""
281 return False
283 @property
284 def isParameter(self):
285 """True if the Parameter under test can be set to an Option or an
286 Argument, False if it only supports one or the other."""
287 return False
289 @property
290 def isChoice(self):
291 """True if the parameter accepts a limited set of input values. Default
292 implementation is to see if the option class has an attribute called
293 choices, which should be of type `list` [`str`]."""
294 return hasattr(self.optionClass, "choices")
296 @property
297 def isBool(self):
298 """True if the option only accepts bool inputs."""
299 return False
301 def makeInputs(self, optionFlag, values=None):
302 """Make the input arguments for a CLI invocation, taking into account
303 if the parameter is an Option (use option flags) or an Argument (do not
304 use option flags).
306 Parameters
307 ----------
308 optionFlag : `str` or `None`
309 The option flag to use if this is an Option. May be None if it is
310 known that the parameter will never be an Option.
311 optionValues : `str`, `list` [`str`], or None
312 The values to use as inputs. If `None`; for an Argument returns an
313 empty list, or for an Option returns a single-item list containing
314 the option flag.
316 Returns
317 -------
318 inputValues : `list` [`str`]
319 The list of values to use as the input parameters to a CLI function
320 invocation.
321 """
322 inputs = []
323 if values is None:
324 self.assertFalse(self.isArgument, "Arguments can not be flag-only; a value is required.")
325 # if there are no values and this is an Option (not an
326 # Argument) then treat it as a click.Flag; the Option will be
327 # True for present, False for not present.
328 inputs.append(optionFlag)
329 return inputs
330 for value in iterable(values):
331 if not self.isArgument:
332 inputs.append(optionFlag)
333 inputs.append(value)
334 return inputs
336 def run_command(self, cmd, args):
337 """Run a command.
339 Parameters
340 ----------
341 cmd : click.Command
342 The command function to call.
343 args : [`str`]
344 The arguments to pass to the function call.
346 Returns
347 -------
348 result : `click.Result`
349 The Result instance that contains the results of the executed
350 command.
351 """
352 return self.runner.invoke(cmd, args)
354 def run_test(self, cmd, cmdArgs, verifyFunc, verifyArgs=None):
355 result = self.run_command(cmd, cmdArgs)
356 verifyFunc(result, verifyArgs)
358 def verifyCalledWith(self, result, mockInfo):
359 """Verify the command function has been called with specified
360 arguments."""
361 self.assertEqual(result.exit_code, 0, clickResultMsg(result))
362 mockInfo.mock.assert_called_with(*mockInfo.expectedArgs, **mockInfo.expectedKwargs)
364 def verifyError(self, result, expectedMsg):
365 """Verify the command failed with a non-zero exit code and an expected
366 output message."""
367 self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
368 self.assertIn(expectedMsg, result.stdout)
370 def verifyMissing(self, result, verifyArgs):
371 """Verify there was a missing argument; that the expected error message
372 has been written to stdout, and that the command exit code is not 0.
373 """
374 self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
375 self.assertRegex(result.stdout, verifyArgs)
378class OptFlagTest(OptTestBase):
379 """A mixin that tests an option behaves as a flag, instead of accepting
380 a value."""
382 def test_forFlag_true(self):
383 helper = CliFactory.mocked(self,
384 expectedKwargs={self.optionKey: True})
385 self.run_test(helper.cli,
386 self.makeInputs(self.optionFlag),
387 self.verifyCalledWith,
388 helper)
390 def test_forFlag_false(self):
391 helper = CliFactory.mocked(self,
392 expectedKwargs={self.optionKey: False})
393 self.run_test(helper.cli,
394 [],
395 self.verifyCalledWith,
396 helper)
399class OptChoiceTest(OptTestBase):
400 """A mixin that tests an option specifies and accepts a list of acceptable
401 choices and rejects choices that are not in that list."""
403 def test_forChoices_validValue(self):
404 """Verify that each valid choice can be passed as a value and is
405 printed to the command line.
406 """
407 for choice, expectedValue in zip(self.choices, self.expectedChoiceValues):
408 helper = CliFactory.mocked(self,
409 expectedKwargs={self.optionKey: expectedValue})
410 self.run_test(helper.cli,
411 self.makeInputs(self.optionFlag, choice),
412 self.verifyCalledWith,
413 helper)
415 def test_forChoices_invalidValue(self):
416 """Verify that an invalid value fails with an expected error message.
417 """
418 cli = CliFactory.noOp(self)
419 choice = self.choices[0]
420 while choice in self.choices:
421 choice += "foo"
422 if self.shortOptionFlag:
423 expected = fr"Invalid value for ['\"]{self.shortOptionFlag}['\"] / ['\"]{self.optionFlag}['\"]"
424 else:
425 expected = fr"Invalid value for ['\"]{self.optionFlag}['\"]"
426 self.run_test(cli, [self.optionFlag, choice], self.verifyMissing, expected)
429class OptCaseInsensitiveTest(OptTestBase):
430 """A mixin that tests an option accepts values in a case-insensitive way.
431 """
433 def test_forCaseInsensitive_upperLower(self):
434 """Verify case insensitivity by making an argument all upper case and
435 all lower case and verifying expected output in both cases."""
436 helper = CliFactory.mocked(self,
437 expectedKwargs={self.optionKey: self.expectedVal})
438 self.run_test(helper.cli,
439 self.makeInputs(self.optionFlag, self.optionValue.upper()),
440 self.verifyCalledWith,
441 helper)
442 self.run_test(helper.cli,
443 self.makeInputs(self.optionFlag, self.optionValue.lower()),
444 self.verifyCalledWith,
445 helper)
448class OptMultipleTest(OptTestBase):
449 """A mixin that tests an option accepts multiple inputs which may be
450 comma separated."""
452 # no need to test multiple=False, this gets tested with the "required"
453 # test case.
455 def test_forMultiple(self):
456 """Test that an option class accepts the 'multiple' keyword and that
457 the command can accept multiple flag inputs for the option, and inputs
458 accept comma-separated values within a single flag argument."""
459 helper = CliFactory.mocked(self,
460 expectedKwargs={self.optionKey: self.expectedMultipleValues},
461 parameterKwargs=dict(multiple=True))
462 self.run_test(helper.cli,
463 self.makeInputs(self.optionFlag, self.optionMultipleValues),
464 self.verifyCalledWith,
465 helper)
467 def test_forMultiple_defaultSingle(self):
468 """Test that the option's 'multiple' argument defaults to False."""
469 helper = CliFactory.mocked(self,
470 expectedKwargs={self.optionKey: self.optionValue})
472 self.run_test(helper.cli,
473 [self.optionValue] if self.isArgument else[self.optionFlag, self.optionValue],
474 self.verifyCalledWith,
475 helper)
478class OptSplitKeyValueTest(OptTestBase):
479 """A mixin that tests that an option that accepts key-value inputs parses
480 those inputs correctly.
481 """
483 def test_forKeyValue(self):
484 """Test multiple key-value inputs and comma separation."""
485 helper = CliFactory.mocked(self,
486 expectedKwargs={self.optionKey: self.expectedMultipleKeyValues},
487 parameterKwargs=dict(multiple=True, split_kv=True))
488 self.run_test(helper.cli,
489 self.makeInputs(self.optionFlag, self.optionMultipleKeyValues),
490 self.verifyCalledWith,
491 helper)
493 def test_forKeyValue_withoutMultiple(self):
494 """Test comma-separated key-value inputs with a parameter that accepts
495 only a single key-value pair."""
497 def verify(result, args):
498 self.assertNotEqual(result.exit_code, 0, clickResultMsg(result))
500 values = ",".join(self.optionMultipleKeyValues)
502 self.run_test(CliFactory.noOp(self, parameterKwargs=dict(split_kv=True)),
503 self.makeInputs(self.optionFlag, values),
504 self.verifyError,
505 f"Error: Could not parse key-value pair '{values}' using separator "
506 f"'{split_kv_separator}', with multiple values not allowed.")
509class OptRequiredTest(OptTestBase):
510 """A mixin that tests that an option that accepts a "required" argument
511 and handles that argument correctly.
512 """
514 def test_required_missing(self):
515 if self.isArgument:
516 expected = fr"Missing argument ['\"]{self.optionName.upper()}['\"]"
517 else:
518 if self.shortOptionFlag:
519 expected = fr"Missing option ['\"]{self.shortOptionFlag}['\"] / ['\"]{self.optionFlag}['\"]"
520 else:
521 expected = fr"Missing option ['\"]\-\-{self.optionName}['\"]"
523 self.run_test(CliFactory.noOp(self, parameterKwargs=dict(required=True)),
524 [],
525 self.verifyMissing,
526 expected)
528 def _test_forRequired_provided(self, required):
529 def doTest(self):
530 helper = CliFactory.mocked(self,
531 expectedKwargs={self.optionKey: self.expectedVal},
532 parameterKwargs=dict(required=required))
533 self.run_test(helper.cli,
534 self.makeInputs(self.optionFlag, self.optionValue),
535 self.verifyCalledWith,
536 helper)
538 if type(self.valueType) == click.Path:
539 OptPathTypeTest.runForPathType(self, doTest)
540 else:
541 doTest(self)
543 def test_required_provided(self):
544 self._test_forRequired_provided(required=True)
546 def test_required_notRequiredProvided(self):
547 self._test_forRequired_provided(required=False)
549 def test_required_notRequiredDefaultValue(self):
550 """Verify that the expected default value is passed for a paramter when
551 it is not used on the command line."""
552 helper = CliFactory.mocked(self,
553 expectedKwargs={self.optionKey: self.expectedValDefault})
554 self.run_test(helper.cli,
555 [],
556 self.verifyCalledWith,
557 helper)
560class OptPathTypeTest(OptTestBase):
561 """A mixin that tests options that have `type=click.Path`.
562 """
564 @staticmethod
565 def runForPathType(testObj, testFunc):
566 """Function to execute the path type test, sets up directories and a
567 file as needed by the test.
569 Parameters
570 ----------
571 testObj : `OptTestBase` instance
572 The `OptTestBase` subclass that is running the test.
573 testFunc : callable
574 The function that executes the test, takes no arguments.
575 """
576 with testObj.runner.isolated_filesystem():
577 if testObj.valueType.exists:
578 # If the file or dir is expected to exist, create it since it's
579 # it doesn't exist because we're running in a temporary
580 # directory.
581 if testObj.valueType.dir_okay:
582 os.makedirs(testObj.optionValue)
583 elif testObj.valueType.file_okay:
584 with open(testObj.optionValue, "w") as _:
585 pass
586 else:
587 testObj.assertTrue(False,
588 "Unexpected; at least one of file_okay or dir_okay should be True.")
589 testFunc(testObj)
591 def test_pathType(self):
592 helper = CliFactory.mocked(self,
593 expectedKwargs={self.optionKey: self.expectedVal})
595 def doTest(self):
596 self.run_test(helper.cli,
597 self.makeInputs(self.optionFlag, self.optionValue),
598 self.verifyCalledWith,
599 helper)
601 OptPathTypeTest.runForPathType(self, doTest)
604class OptHelpTest(OptTestBase):
605 """A mixin that tests that an option has a defaultHelp parameter, accepts
606 a custom help paramater, and prints the help message correctly.
607 """
608 # Specifying a very wide terminal prevents Click from wrapping text when
609 # rendering output, which causes issues trying to compare expected strings.
610 wideTerminal = dict(context_settings=dict(terminal_width=1000000))
612 def _verify_forHelp(self, result, expectedHelpText):
613 self.assertEqual(result.exit_code, 0, clickResultMsg(result))
614 self.assertIn(expectedHelpText, result.output)
616 def test_help_default(self):
617 self.run_test(CliFactory.noOp(self, cmdInitKwArgs=self.wideTerminal),
618 ["--help"],
619 self._verify_forHelp,
620 self.optionClass.defaultHelp)
622 def test_help_custom(self):
623 helpText = "foobarbaz"
624 self.run_test(CliFactory.noOp(self,
625 parameterKwargs=dict(help=helpText),
626 cmdInitKwArgs=self.wideTerminal),
627 ["--help"],
628 self._verify_forHelp,
629 helpText)
631 def test_help_optionMetavar(self):
632 """Test that a specified metavar prints correctly in the help output
633 for Options."""
635 # For now only run on test cases that define the metavar to test.
636 # This could be expanded to get the raw metavar out of the parameter
637 # and test for expected formatting of all shared option metavars.
638 if self.metavar is None:
639 return
641 def getMetavar(isRequired):
642 return self.metavar if isRequired else f"[{self.metavar}]"
644 parameters = inspect.signature(self.optionClass.__init__).parameters.values()
645 supportedInitArgs = [parameter.name for parameter in parameters]
647 def doTest(required, multiple):
648 """Test for the expected parameter flag(s), metavar, and muliptle
649 indicator in the --help output.
651 Parameters
652 ----------
653 required : `bool` or None
654 True if the parameter is required, False if it is not required,
655 or None if the parameter initializer does not take a required
656 argument, in which case it is treated as not required.
657 multiple : `bool` or None
658 True if the parameter accepts multiple inputs, False if it does
659 not, or None if the parameter initializer does not take a
660 multiple argument, in which case it is treated as not multiple.
661 """
662 if self.isArgument:
663 expected = f"{getMetavar(required)}{' ...' if multiple else ''}"
664 else:
665 if self.shortOptionFlag is not None and self.optionFlag is not None:
666 expected = ", ".join([self.shortOptionFlag, self.optionFlag])
667 elif (self.shortOptionFlag is not None):
668 expected = self.shortOptionFlag
669 else:
670 expected = self.optionFlag
671 expected = f"{expected} {self.metavar}{' ...' if multiple else ''}"
672 parameterKwargs = parameterKwargs = dict(required=required)
673 if multiple is not None:
674 parameterKwargs["multiple"] = multiple
675 if "metavar" in supportedInitArgs:
676 parameterKwargs["metavar"] = self.metavar
677 self.run_test(CliFactory.noOp(self, parameterKwargs=parameterKwargs),
678 ["--help"],
679 self._verify_forHelp,
680 expected)
682 for required in (False, True) if "required" in supportedInitArgs else (None,):
683 for multiple in (False, True) if "multiple" in supportedInitArgs else (None,):
684 doTest(required, multiple)