23 """Support code for running unit tests""" 37 __all__ = [
"init",
"run",
"MemoryTestCase",
"ExecutablesTestCase",
"getTempFilePath",
38 "TestCase",
"assertFloatsAlmostEqual",
"assertFloatsNotEqual",
"assertFloatsEqual"]
47 import lsst.daf.base
as dafBase
61 def _get_open_files():
62 """Return a set containing the list of files currently open in this 68 Set containing the list of open files. 72 return set(p.path
for p
in psutil.Process().
open_files())
76 """Initialize the memory tester and file descriptor leak tester.""" 80 open_files = _get_open_files()
83 def run(suite, exit=True):
84 """Run a test suite and report the test return status to caller or shell. 86 .. note:: Deprecated in 13_0 87 Use `unittest.main()` instead, which automatically detects 88 all tests in a test case and does not require a test suite. 92 suite : `unittest.TestSuite` 94 exit : `bool`, optional 95 If `True`, Python process will exit with the test exit status. 100 If ``exit`` is `False`, will return 0 if the tests passed, or 1 if 104 warnings.warn(
"lsst.utils.tests.run() is deprecated; please use unittest.main() instead",
105 DeprecationWarning, stacklevel=2)
107 if unittest.TextTestRunner().
run(suite).wasSuccessful():
119 """Sort supplied test suites such that MemoryTestCases are at the end. 121 `lsst.utils.tests.MemoryTestCase` tests should always run after any other 127 Sequence of test suites. 131 suite : `unittest.TestSuite` 132 A combined `~unittest.TestSuite` with 133 `~lsst.utils.tests.MemoryTestCase` at the end. 136 suite = unittest.TestSuite()
138 for test_suite
in tests:
145 for method
in test_suite:
146 bases = inspect.getmro(method.__class__)
148 if bases
is not None and MemoryTestCase
in bases:
149 memtests.append(test_suite)
151 suite.addTests(test_suite)
153 if isinstance(test_suite, MemoryTestCase):
154 memtests.append(test_suite)
156 suite.addTest(test_suite)
157 suite.addTests(memtests)
168 unittest.defaultTestLoader.suiteClass = suiteClassWrapper
172 """Check for memory leaks since memId0 was allocated""" 179 """Reset the leak counter when the tests have been completed""" 183 """Check if any file descriptors are open since init() called.""" 185 self.skipTest(
"Unable to test file descriptor leaks. psutil unavailable.")
188 now_open = _get_open_files()
191 now_open = set(f
for f
in now_open
if not f.endswith(
".car")
and 192 not f.startswith(
"/proc/")
and 193 not f.endswith(
".ttf")
and 194 not (f.startswith(
"/var/lib/")
and f.endswith(
"/passwd"))
and 195 not f.endswith(
"astropy.log"))
197 diff = now_open.difference(open_files)
200 print(
"File open: %s" % f)
201 self.fail(
"Failed to close %d file%s" % (len(diff),
"s" if len(diff) != 1
else ""))
205 """Test that executables can be run and return good status. 207 The test methods are dynamically created. Callers 208 must subclass this class in their own test file and invoke 209 the create_executable_tests() class method to register the tests. 211 TESTS_DISCOVERED = -1
215 """Abort testing if automated test creation was enabled and 216 no tests were found.""" 219 raise Exception(
"No executables discovered.")
222 """This test exists to ensure that there is at least one test to be 223 executed. This allows the test runner to trigger the class set up 224 machinery to test whether there are some executables to test.""" 228 """Check an executable runs and returns good status. 230 Prints output to standard out. On bad exit status the test 231 fails. If the executable can not be located the test is skipped. 236 Path to an executable. ``root_dir`` is not used if this is an 238 root_dir : `str`, optional 239 Directory containing executable. Ignored if `None`. 240 args : `list` or `tuple`, optional 241 Arguments to be provided to the executable. 242 msg : `str`, optional 243 Message to use when the test fails. Can be `None` for default 249 The executable did not return 0 exit status. 252 if root_dir
is not None and not os.path.isabs(executable):
253 executable = os.path.join(root_dir, executable)
256 sp_args = [executable]
257 argstr =
"no arguments" 260 argstr =
'arguments "' +
" ".join(args) +
'"' 262 print(
"Running executable '{}' with {}...".format(executable, argstr))
263 if not os.path.exists(executable):
264 self.skipTest(
"Executable {} is unexpectedly missing".format(executable))
267 output = subprocess.check_output(sp_args)
268 except subprocess.CalledProcessError
as e:
270 failmsg =
"Bad exit status from '{}': {}".format(executable, e.returncode)
271 print(output.decode(
'utf-8'))
278 def _build_test_method(cls, executable, root_dir):
279 """Build a test method and attach to class. 281 A test method is created for the supplied excutable located 282 in the supplied root directory. This method is attached to the class 283 so that the test runner will discover the test and run it. 288 The class in which to create the tests. 290 Name of executable. Can be absolute path. 292 Path to executable. Not used if executable path is absolute. 294 if not os.path.isabs(executable):
295 executable = os.path.abspath(os.path.join(root_dir, executable))
298 test_name =
"test_exe_" + executable.replace(
"/",
"_")
301 def test_executable_runs(*args):
303 self.assertExecutable(executable)
306 test_executable_runs.__name__ = test_name
307 setattr(cls, test_name, test_executable_runs)
311 """Discover executables to test and create corresponding test methods. 313 Scans the directory containing the supplied reference file 314 (usually ``__file__`` supplied from the test class) to look for 315 executables. If executables are found a test method is created 316 for each one. That test method will run the executable and 317 check the returned value. 319 Executable scripts with a ``.py`` extension and shared libraries 320 are ignored by the scanner. 322 This class method must be called before test discovery. 327 Path to a file within the directory to be searched. 328 If the files are in the same location as the test file, then 329 ``__file__`` can be used. 330 executables : `list` or `tuple`, optional 331 Sequence of executables that can override the automated 332 detection. If an executable mentioned here is not found, a 333 skipped test will be created for it, rather than a failed 338 >>> cls.create_executable_tests(__file__) 342 ref_dir = os.path.abspath(os.path.dirname(ref_file))
344 if executables
is None:
347 for root, dirs, files
in os.walk(ref_dir):
350 if not f.endswith(
".py")
and not f.endswith(
".so"):
351 full_path = os.path.join(root, f)
352 if os.access(full_path, os.X_OK):
353 executables.append(full_path)
362 for e
in executables:
366 @contextlib.contextmanager
368 """Return a path suitable for a temporary file and try to delete the 371 If the with block completes successfully then the file is deleted, 372 if possible; failure results in a printed warning. 373 If a file is remains when it should not, a RuntimeError exception is 374 raised. This exception is also raised if a file is not present on context 375 manager exit when one is expected to exist. 376 If the block exits with an exception the file if left on disk so it can be 377 examined. The file name has a random component such that nested context 378 managers can be used with the same file suffix. 384 File name extension, e.g. ``.fits``. 385 expectOutput : `bool`, optional 386 If `True`, a file should be created within the context manager. 387 If `False`, a file should not be present when the context manager 393 Path for a temporary file. The path is a combination of the caller's 394 file path and the name of the top-level function 400 # file tests/testFoo.py 402 import lsst.utils.tests 403 class FooTestCase(unittest.TestCase): 404 def testBasics(self): 408 with lsst.utils.tests.getTempFilePath(".fits") as tmpFile: 409 # if tests/.tests exists then 410 # tmpFile = "tests/.tests/testFoo_testBasics.fits" 411 # otherwise tmpFile = "testFoo_testBasics.fits" 413 # at the end of this "with" block the path tmpFile will be 414 # deleted, but only if the file exists and the "with" 415 # block terminated normally (rather than with an exception) 418 stack = inspect.stack()
420 for i
in range(2, len(stack)):
421 frameInfo = inspect.getframeinfo(stack[i][0])
423 callerFilePath = frameInfo.filename
424 callerFuncName = frameInfo.function
425 elif callerFilePath == frameInfo.filename:
427 callerFuncName = frameInfo.function
431 callerDir, callerFileNameWithExt = os.path.split(callerFilePath)
432 callerFileName = os.path.splitext(callerFileNameWithExt)[0]
433 outDir = os.path.join(callerDir,
".tests")
434 if not os.path.isdir(outDir):
436 prefix =
"%s_%s-" % (callerFileName, callerFuncName)
437 outPath = tempfile.mktemp(dir=outDir, suffix=ext, prefix=prefix)
438 if os.path.exists(outPath):
441 warnings.warn(
"Unexpectedly found pre-existing tempfile named %r" % (outPath,),
450 fileExists = os.path.exists(outPath)
453 raise RuntimeError(
"Temp file expected named {} but none found".format(outPath))
456 raise RuntimeError(
"Unexpectedly discovered temp file named {}".format(outPath))
463 warnings.warn(
"Warning: could not remove file %r: %s" % (outPath, e), stacklevel=3)
467 """Subclass of unittest.TestCase that adds some custom assertions for 473 """A decorator to add a free function to our custom TestCase class, while also 474 making it available as a free function. 476 setattr(TestCase, func.__name__, func)
482 """.. note:: Deprecated in 12_0""" 483 warnings.warn(
"assertRaisesLsstCpp is deprecated; please just use TestCase.assertRaises",
484 DeprecationWarning, stacklevel=2)
485 return testcase.assertRaises(excClass, callableObj, *args, **kwargs)
489 """Decorator to enter the debugger when there's an uncaught exception 491 To use, just slap a ``@debugger()`` on your function. 493 You may provide specific exception classes to catch as arguments to 494 the decorator function, e.g., 495 ``@debugger(RuntimeError, NotImplementedError)``. 496 This defaults to just `AssertionError`, for use on `unittest.TestCase` 499 Code provided by "Rosh Oxymoron" on StackOverflow: 500 http://stackoverflow.com/questions/4398967/python-unit-testing-automatically-running-the-debugger-when-a-test-fails 504 Consider using ``pytest --pdb`` instead of this decorator. 507 exceptions = (AssertionError, )
511 def wrapper(*args, **kwargs):
513 return f(*args, **kwargs)
517 pdb.post_mortem(sys.exc_info()[2])
523 """Plot the comparison of two 2-d NumPy arrays. 527 lhs : `numpy.ndarray` 528 LHS values to compare; a 2-d NumPy array 529 rhs : `numpy.ndarray` 530 RHS values to compare; a 2-d NumPy array 531 bad : `numpy.ndarray` 532 A 2-d boolean NumPy array of values to emphasize in the plots 533 diff : `numpy.ndarray` 534 difference array; a 2-d NumPy array, or None to show lhs-rhs 536 Filename to save the plot to. If None, the plot will be displayed in 541 This method uses `matplotlib` and imports it internally; it should be 542 wrapped in a try/except block within packages that do not depend on 543 `matplotlib` (including `~lsst.utils`). 545 from matplotlib
import pyplot
551 badImage = numpy.zeros(bad.shape + (4,), dtype=numpy.uint8)
552 badImage[:, :, 0] = 255
553 badImage[:, :, 1] = 0
554 badImage[:, :, 2] = 0
555 badImage[:, :, 3] = 255*bad
556 vmin1 = numpy.minimum(numpy.min(lhs), numpy.min(rhs))
557 vmax1 = numpy.maximum(numpy.max(lhs), numpy.max(rhs))
558 vmin2 = numpy.min(diff)
559 vmax2 = numpy.max(diff)
560 for n, (image, title)
in enumerate([(lhs,
"lhs"), (rhs,
"rhs"), (diff,
"diff")]):
561 pyplot.subplot(2, 3, n + 1)
562 im1 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
563 vmin=vmin1, vmax=vmax1)
565 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
568 pyplot.subplot(2, 3, n + 4)
569 im2 = pyplot.imshow(image, cmap=pyplot.cm.gray, interpolation=
'nearest', origin=
'lower',
570 vmin=vmin2, vmax=vmax2)
572 pyplot.imshow(badImage, alpha=0.2, interpolation=
'nearest', origin=
'lower')
575 pyplot.subplots_adjust(left=0.05, bottom=0.05, top=0.92, right=0.75, wspace=0.05, hspace=0.05)
576 cax1 = pyplot.axes([0.8, 0.55, 0.05, 0.4])
577 pyplot.colorbar(im1, cax=cax1)
578 cax2 = pyplot.axes([0.8, 0.05, 0.05, 0.4])
579 pyplot.colorbar(im2, cax=cax2)
581 pyplot.savefig(plotFileName)
588 atol=sys.float_info.epsilon, relTo=None,
589 printFailures=True, plotOnFailure=False,
590 plotFileName=None, invert=False, msg=None):
591 """Highly-configurable floating point comparisons for scalars and arrays. 593 The test assertion will fail if all elements ``lhs`` and ``rhs`` are not 594 equal to within the tolerances specified by ``rtol`` and ``atol``. 595 More precisely, the comparison is: 597 ``abs(lhs - rhs) <= relTo*rtol OR abs(lhs - rhs) <= atol`` 599 If ``rtol`` or ``atol`` is `None`, that term in the comparison is not 602 When not specified, ``relTo`` is the elementwise maximum of the absolute 603 values of ``lhs`` and ``rhs``. If set manually, it should usually be set 604 to either ``lhs`` or ``rhs``, or a scalar value typical of what is 609 testCase : `unittest.TestCase` 610 Instance the test is part of. 611 lhs : scalar or array-like 612 LHS value(s) to compare; may be a scalar or array-like of any 614 rhs : scalar or array-like 615 RHS value(s) to compare; may be a scalar or array-like of any 617 rtol : `float`, optional 618 Relative tolerance for comparison; defaults to double-precision 620 atol : `float`, optional 621 Absolute tolerance for comparison; defaults to double-precision 623 relTo : `float`, optional 624 Value to which comparison with rtol is relative. 625 printFailures : `bool`, optional 626 Upon failure, print all inequal elements as part of the message. 627 plotOnFailure : `bool`, optional 628 Upon failure, plot the originals and their residual with matplotlib. 629 Only 2-d arrays are supported. 630 plotFileName : `str`, optional 631 Filename to save the plot to. If `None`, the plot will be displayed in 633 invert : `bool`, optional 634 If `True`, invert the comparison and fail only if any elements *are* 635 equal. Used to implement `~lsst.utils.tests.assertFloatsNotEqual`, 636 which should generally be used instead for clarity. 637 msg : `str`, optional 638 String to append to the error message when assert fails. 643 The values are not almost equal. 645 if not numpy.isfinite(lhs).all():
646 testCase.fail(
"Non-finite values in lhs")
647 if not numpy.isfinite(rhs).all():
648 testCase.fail(
"Non-finite values in rhs")
650 absDiff = numpy.abs(lhs - rhs)
653 relTo = numpy.maximum(numpy.abs(lhs), numpy.abs(rhs))
655 relTo = numpy.abs(relTo)
656 bad = absDiff > rtol*relTo
658 bad = numpy.logical_and(bad, absDiff > atol)
661 raise ValueError(
"rtol and atol cannot both be None")
663 failed = numpy.any(bad)
666 bad = numpy.logical_not(bad)
668 failStr =
"are the same" 674 if numpy.isscalar(bad):
676 errMsg = [
"%s %s %s; diff=%s with atol=%s" 677 % (lhs, cmpStr, rhs, absDiff, atol)]
679 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s" 680 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol)]
682 errMsg = [
"%s %s %s; diff=%s/%s=%s with rtol=%s, atol=%s" 683 % (lhs, cmpStr, rhs, absDiff, relTo, absDiff/relTo, rtol, atol)]
685 errMsg = [
"%d/%d elements %s with rtol=%s, atol=%s" 686 % (bad.sum(), bad.size, failStr, rtol, atol)]
688 if len(lhs.shape) != 2
or len(rhs.shape) != 2:
689 raise ValueError(
"plotOnFailure is only valid for 2-d arrays")
691 plotImageDiff(lhs, rhs, bad, diff=diff, plotFileName=plotFileName)
693 errMsg.append(
"Failure plot requested but matplotlib could not be imported.")
698 if numpy.isscalar(relTo):
699 relTo = numpy.ones(bad.shape, dtype=float) * relTo
700 if numpy.isscalar(lhs):
701 lhs = numpy.ones(bad.shape, dtype=float) * lhs
702 if numpy.isscalar(rhs):
703 rhs = numpy.ones(bad.shape, dtype=float) * rhs
705 for a, b, diff
in zip(lhs[bad], rhs[bad], absDiff[bad]):
706 errMsg.append(
"%s %s %s (diff=%s)" % (a, cmpStr, b, diff))
708 for a, b, diff, rel
in zip(lhs[bad], rhs[bad], absDiff[bad], relTo[bad]):
709 errMsg.append(
"%s %s %s (diff=%s/%s=%s)" % (a, cmpStr, b, diff, rel, diff/rel))
713 testCase.assertFalse(failed, msg=
"\n".join(errMsg))
718 """Fail a test if the given floating point values are equal to within the 721 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with 722 ``rtol=atol=0``) for more information. 726 testCase : `unittest.TestCase` 727 Instance the test is part of. 728 lhs : scalar or array-like 729 LHS value(s) to compare; may be a scalar or array-like of any 731 rhs : scalar or array-like 732 RHS value(s) to compare; may be a scalar or array-like of any 738 The values are almost equal. 746 Assert that lhs == rhs (both numeric types, whether scalar or array). 748 See `~lsst.utils.tests.assertFloatsAlmostEqual` (called with 749 ``rtol=atol=0``) for more information. 753 testCase : `unittest.TestCase` 754 Instance the test is part of. 755 lhs : scalar or array-like 756 LHS value(s) to compare; may be a scalar or array-like of any 758 rhs : scalar or array-like 759 RHS value(s) to compare; may be a scalar or array-like of any 765 The values are not equal. 772 """.. note:: Deprecated in 12_0""" 773 warnings.warn(
"assertClose is deprecated; please use TestCase.assertFloatsAlmostEqual",
774 DeprecationWarning, stacklevel=2)
780 """.. note:: Deprecated in 12_0""" 781 warnings.warn(
"assertNotClose is deprecated; please use TestCase.assertFloatsNotEqual",
782 DeprecationWarning, stacklevel=2)
def suiteClassWrapper(tests)
def assertExecutable(self, executable, root_dir=None, args=None, msg=None)
def assertFloatsEqual(testCase, lhs, rhs, kwargs)
def plotImageDiff(lhs, rhs, bad=None, diff=None, plotFileName=None)
def assertClose(args, kwargs)
def _build_test_method(cls, executable, root_dir)
def assertFloatsAlmostEqual(testCase, lhs, rhs, rtol=sys.float_info.epsilon, atol=sys.float_info.epsilon, relTo=None, printFailures=True, plotOnFailure=False, plotFileName=None, invert=False, msg=None)
def assertFloatsNotEqual(testCase, lhs, rhs, kwds)
def run(suite, exit=True)
def getTempFilePath(ext, expectOutput=True)
def assertNotClose(args, kwargs)
def create_executable_tests(cls, ref_file, executables=None)
def testFileDescriptorLeaks(self)
def assertRaisesLsstCpp(testcase, excClass, callableObj, args, kwargs)