Coverage for python/lsst/sconsUtils/tests.py: 8%
173 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-22 11:01 +0000
« prev ^ index » next coverage.py v6.5.0, created at 2023-02-22 11:01 +0000
1"""Control which tests run, and how.
2"""
4__all__ = ("Control", )
6import glob
7import os
8import sys
9import pipes
10import SCons.Script
11from . import state
12from . import utils
15class Control:
16 """A class to control and run unit tests.
18 This class is unchanged from previous versions of sconsUtils, but it will
19 now generally be called via
20 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
22 Parameters
23 ----------
24 env : `SCons.Environment`
25 An SCons Environment (almost always `lsst.sconsUtils.env`).
26 ignoreList : `list`, optional
27 A list of tests that should NOT be run --- useful in conjunction
28 with glob patterns. If a file is listed as "@fileName", the @ is
29 stripped and we don't bother to check if fileName exists (useful for
30 machine-generated files).
31 expectedFailures : `dict`, optional
32 A dictionary; the keys are tests that are known to fail; the values
33 are strings to print.
34 args : `dict`, optional
35 A dictionary with testnames as keys, and argument strings as values.
36 As scons always runs from the top-level directory, tests has to fiddle
37 with paths. If an argument is a file this is done automatically; if
38 it's e.g., just a basename then you have to tell tests that it's
39 really (part of a) filename by prefixing the name by ``file:``.
40 tmpDir : `str`, optional
41 The location of the test outputs.
42 verbose : `bool`, optional
43 How chatty you want the test code to be.
45 Notes
46 -----
47 Sample usage:
49 .. code-block:: python
51 tests = lsst.tests.Control(
52 env,
53 args={
54 "MaskIO_1" : "data/871034p_1_MI_msk.fits",
55 "MaskedImage_1" : "file:data/871034p_1_MI foo",
56 },
57 ignoreList=["Measure_1"],
58 expectedFailures={"BBox_1": "Problem with single-pixel BBox"}
59 )
61 This class is unchanged from previous versions of sconsUtils, but it will
62 now generally be called via
63 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
64 """
66 _IGNORE = "IGNORE"
67 _EXPECT_FAILURE = "EXPECT_FAILURE"
69 def __init__(self, env, ignoreList=None, expectedFailures=None, args=None,
70 tmpDir=".tests", verbose=False):
72 # Need to define our own Astropy cache directories.
73 # Unfortunately we can not simply set XDG_CACHE_HOME
74 # to $HOME/.astropy. Do not forward $HOME or the XDG_CONFIG_HOME
75 # environment variables since those may affect test outcomes.
76 xdgCacheVar = "XDG_CACHE_HOME"
77 if xdgCacheVar not in os.environ:
78 if "~" in os.path.expanduser("~"):
79 state.log.warn(f"Neither $HOME nor ${xdgCacheVar} defined. No Astropy cache enabled.")
80 else:
81 # We need a directory for the cache and that directory
82 # has to have an "astropy" directory inside it. We can
83 # use ~/.astropy or ~/.lsst or a tmp directory but choose
84 # ~/.lsst initially.
85 cacheDir = os.path.expanduser("~/.lsst")
86 astropyCacheDir = os.path.join(cacheDir, "astropy")
87 if not os.path.exists(astropyCacheDir):
88 os.makedirs(astropyCacheDir, exist_ok=True) # Race condition is okay
89 os.environ[xdgCacheVar] = cacheDir
90 else:
91 if not os.path.exists(os.path.expanduser(os.path.join(os.environ[xdgCacheVar],
92 "astropy"))):
93 state.log.warn(f"{xdgCacheVar} is set but will not be used for "
94 "astropy due to lack of astropy directory within it")
96 # Forward some environment to the tests
97 for envvar in ["PYTHONPATH", "HTTP_PROXY", "HTTPS_PROXY", xdgCacheVar]:
98 if envvar in os.environ:
99 env.AppendENVPath(envvar, os.environ[envvar])
101 self._env = env
103 self._tmpDir = tmpDir
104 self._cwd = os.path.abspath(os.path.curdir)
106 # Calculate the absolute path for temp dir if it is relative.
107 # This assumes the temp dir is relative to where the tests SConscript
108 # file is located. SCons will know how to handle this itself but
109 # some options require the code to know where to write things.
110 if os.path.isabs(self._tmpDir):
111 self._tmpDirAbs = self._tmpDir
112 else:
113 self._tmpDirAbs = os.path.join(self._cwd, self._tmpDir)
115 self._verbose = verbose
117 self._info = {} # information about processing targets
118 if ignoreList:
119 for f in ignoreList:
120 if f.startswith("@"): # @dfilename => don't complain if filename doesn't exist
121 f = f[1:]
122 else:
123 if not os.path.exists(f):
124 state.log.warn("You're ignoring a non-existent file, %s" % f)
125 self._info[f] = (self._IGNORE, None)
127 if expectedFailures:
128 for f in expectedFailures:
129 self._info[f] = (self._EXPECT_FAILURE, expectedFailures[f])
131 if args:
132 self._args = args # arguments for tests
133 else:
134 self._args = {}
136 self.runExamples = True # should I run the examples?
137 try:
138 # file is user read/write/executable
139 self.runExamples = (os.stat(self._tmpDir).st_mode & 0o700) != 0
140 except OSError:
141 pass
143 if not self.runExamples:
144 print("Not running examples; \"chmod 755 %s\" to run them again" % self._tmpDir,
145 file=sys.stderr)
147 def args(self, test):
148 """Arguments to use for this test.
150 Parameters
151 ----------
152 test : `str`
153 Test file to be run.
155 Returns
156 -------
157 args : `str`
158 The arguments as a single string. An empty string is returned
159 if no arguments were specified in the constructor.
160 """
161 try:
162 return self._args[test]
163 except KeyError:
164 return ""
166 def ignore(self, test):
167 """Should the test be ignored.
169 Parameters
170 ----------
171 test : `str`
172 The test target name.
174 Returns
175 -------
176 ignore : `bool`
177 Whether the test should be ignored or not.
178 """
179 if not test.endswith(".py") and \
180 len(self._env.Glob(test)) == 0: # we don't know how to build it
181 return True
183 ignoreFile = test in self._info and self._info[test][0] == self._IGNORE
185 if self._verbose and ignoreFile:
186 print("Skipping", test, file=sys.stderr)
188 return ignoreFile
190 def messages(self, test):
191 """Return the messages to be used in case of success/failure.
193 Parameters
194 ----------
195 test : `str`
196 The test target.
198 Returns
199 -------
200 messages : `tuple`
201 A `tuple` containing four strings: whether the test should pass
202 (as a value "true" or "false") and the associated message, and
203 whether the test should fail and the associated message.
204 """
206 if test in self._info and self._info[test][0] == self._EXPECT_FAILURE:
207 msg = self._info[test][1]
208 return ("false", "Passed, but should have failed: %s" % msg,
209 "true", "Failed as expected: %s" % msg)
210 else:
211 return ("true", "passed",
212 "false", "failed")
214 def run(self, fileGlob):
215 """Create a test target for each file matching the supplied glob.
217 Parameters
218 ----------
219 fileGlob : `str` or `SCons.Environment.Glob`
220 File matching glob.
222 Returns
223 -------
224 targets :
225 Test target for each matching file.
226 """
228 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
229 fileGlob = str(fileGlob)
230 targets = []
231 if not self.runExamples:
232 return targets
234 # Determine any library load path values that we have to prepend
235 # to the command.
236 libpathstr = utils.libraryLoaderEnvironment()
238 for f in glob.glob(fileGlob):
239 interpreter = "" # interpreter to run test, if needed
241 if f.endswith(".cc"): # look for executable
242 f = os.path.splitext(f)[0]
243 else:
244 interpreter = "pytest -Wd --durations=5 --junit-xml=${TARGET}.xml"
245 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
246 interpreter += " --log-level=DEBUG"
247 interpreter += self._getPytestCoverageCommand()
249 if self.ignore(f):
250 continue
252 target = os.path.join(self._tmpDir, f)
254 args = []
255 for a in self.args(f).split(" "):
256 # if a is a file, make it an absolute name as scons runs from
257 # the root directory
258 filePrefix = "file:"
259 if a.startswith(filePrefix): # they explicitly said that this was a file
260 a = os.path.join(self._cwd, a[len(filePrefix):])
261 else:
262 try: # see if it's a file
263 os.stat(a)
264 a = os.path.join(self._cwd, a)
265 except OSError:
266 pass
268 args += [a]
270 (should_pass, passedMsg, should_fail, failedMsg) = self.messages(f)
272 expandedArgs = " ".join(args)
273 result = self._env.Command(target, f, """
274 @rm -f ${TARGET}.failed;
275 @printf "%%s" 'running ${SOURCES}... ';
276 @echo $SOURCES %s > $TARGET; echo >> $TARGET;
277 @if %s %s $SOURCES %s >> $TARGET 2>&1; then \
278 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
279 echo "%s"; \
280 else \
281 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
282 echo "%s"; \
283 fi;
284 """ % (expandedArgs, libpathstr, interpreter, expandedArgs, should_pass,
285 passedMsg, should_fail, failedMsg))
287 targets.extend(result)
289 self._env.Alias(os.path.basename(target), target)
291 self._env.Clean(target, self._tmpDir)
293 return targets
295 def runPythonTests(self, pyList):
296 """Add a single target for testing all python files.
298 Parameters
299 ----------
300 pyList : `list`
301 A list of nodes corresponding to python test files. The
302 IgnoreList is respected when scanning for entries. If pyList
303 is `None`, or an empty list, it uses automated test discovery
304 within pytest. This differs from the behavior of
305 `lsst.sconsUtils.BasicSconscript.tests`
306 where a distinction is made.
308 Returns
309 -------
310 target : `list`
311 Returns a list containing a single target.
312 """
314 if pyList is None:
315 pyList = []
317 # Determine any library load path values that we have to prepend
318 # to the command.
319 libpathstr = utils.libraryLoaderEnvironment()
321 # Get list of python files with the path included.
322 pythonTestFiles = []
323 for fileGlob in pyList:
324 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
325 fileGlob = str(fileGlob)
326 for f in glob.glob(fileGlob):
327 if self.ignore(f):
328 continue
329 pythonTestFiles.append(os.path.join(self._cwd, f))
331 # Now set up the python testing target
332 # We always want to run this with the tests target.
333 # We have decided to use pytest caching so that on reruns we only
334 # run failed tests.
335 lfnfOpt = "none" if 'install' in SCons.Script.COMMAND_LINE_TARGETS else "all"
336 interpreter = f"pytest -Wd --lf --lfnf={lfnfOpt}"
337 interpreter += " --durations=5 --junit-xml=${TARGET} --session2file=${TARGET}.out"
338 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
339 interpreter += " --log-level=DEBUG"
340 interpreter += self._getPytestCoverageCommand()
342 # Ignore doxygen build directories since they can confuse pytest
343 # test collection
344 interpreter += " --ignore=doc/html --ignore=doc/xml"
346 # Ignore the C++ directories since they will never have python
347 # code and doing this will speed up test collection
348 interpreter += " --ignore=src --ignore=include --ignore=lib"
350 # Ignore the eups directory
351 interpreter += " --ignore=ups"
353 # We currently have a race condition in test collection when
354 # examples has C++ code. Removing it from the scan will get us through
355 # until we can fix the problem properly. Rely on GitHub PRs to
356 # do the flake8 check.
357 interpreter += " --ignore=examples"
359 # Also include temporary files made by compilers.
360 # These can come from examples directories that include C++.
361 interpreter += " --ignore-glob='*.tmp'"
363 target = os.path.join(self._tmpDir, "pytest-{}.xml".format(self._env['eupsProduct']))
365 # Work out how many jobs scons has been configured to use
366 # and use that number with pytest. This could cause trouble
367 # if there are lots of binary tests to run and lots of singles.
368 njobs = self._env.GetOption("num_jobs")
369 print("Running pytest with {} process{}".format(njobs, "" if njobs == 1 else "es"))
370 if njobs > 1:
371 # We unambiguously specify the Python interpreter to be used to
372 # execute tests. This ensures that all pytest-xdist worker
373 # processes refer to the same Python as the xdist controller, and
374 # hence avoids pollution of ``sys.path`` that can happen when we
375 # call the same interpreter by different paths (for example, if
376 # the controller process calls ``miniconda/bin/python``, and the
377 # workers call ``current/bin/python``, the workers will end up
378 # with site-packages directories corresponding to both locations
379 # on ``sys.path``, even if the one is a symlink to the other).
380 executable = os.path.realpath(sys.executable)
382 # if there is a space in the executable path we have to use the
383 # original method and hope things work okay. This will be rare but
384 # without this a space in the path is impossible because of how
385 # xdist currently parses the tx option
386 interpreter = interpreter + " --max-worker-restart=0"
387 if " " not in executable:
388 interpreter = (interpreter
389 + " -d --tx={}*popen//python={}".format(njobs, executable))
390 else:
391 interpreter = interpreter + " -n {}".format(njobs)
393 # Remove target so that we always trigger pytest
394 if os.path.exists(target):
395 os.unlink(target)
397 if not pythonTestFiles:
398 print("pytest: automated test discovery mode enabled.")
399 else:
400 nfiles = len(pythonTestFiles)
401 print("pytest: running on {} Python test file{}.".format(nfiles, "" if nfiles == 1 else "s"))
403 # If we ran all the test, then copy the previous test
404 # execution products to `.all' files so we can retrieve later.
405 # If we skip the test (exit code 5), retrieve those `.all' files.
406 cmd = ""
407 if lfnfOpt == "all":
408 cmd += "@rm -f ${{TARGET}} ${{TARGET}}.failed;"
409 cmd += """
410 @printf "%s\\n" 'running global pytest... ';
411 @({2} {0} {1}); \
412 export rc="$?"; \
413 if [ "$$rc" -eq 0 ]; then \
414 echo "Global pytest run completed successfully"; \
415 cp ${{TARGET}} ${{TARGET}}.all || true; \
416 cp ${{TARGET}}.out ${{TARGET}}.out.all || true; \
417 elif [ "$$rc" -eq 5 ]; then \
418 echo "Global pytest run completed successfully - no tests ran"; \
419 mv ${{TARGET}}.all ${{TARGET}} || true; \
420 mv ${{TARGET}}.out.all ${{TARGET}}.out || true; \
421 else \
422 echo "Global pytest run: failed with $$rc"; \
423 mv ${{TARGET}}.out ${{TARGET}}.failed; \
424 fi;
425 """
426 testfiles = " ".join([pipes.quote(p) for p in pythonTestFiles])
427 result = self._env.Command(target, None, cmd.format(interpreter, testfiles, libpathstr))
429 self._env.Alias(os.path.basename(target), target)
430 self._env.Clean(target, self._tmpDir)
432 return [result]
434 def junitPrefix(self):
435 """Calculate the prefix to use for the JUnit output.
437 Returns
438 -------
439 prefix : `str`
440 Prefix string to use.
442 Notes
443 -----
444 Will use the EUPS product being built and the value of the
445 ``LSST_JUNIT_PREFIX`` environment variable if that is set.
446 """
447 controlVar = "LSST_JUNIT_PREFIX"
448 prefix = self._env['eupsProduct']
450 if controlVar in os.environ:
451 prefix += ".{0}".format(os.environ[controlVar])
453 return prefix
455 def _getPytestCoverageCommand(self):
456 """Form the additional arguments required to enable coverage testing.
458 Coverage output files are written using ``${TARGET}`` as a base.
460 Returns
461 -------
462 options : `str`
463 String defining the coverage-specific arguments to give to the
464 pytest command.
465 """
467 options = ""
469 # Basis for deriving file names
470 # We use the magic target from SCons.
471 prefix = "${TARGET}"
473 # Only report coverage for files in the build tree.
474 # If --cov is used full coverage will be reported for all installed
475 # code as well, but that is probably a distraction as for this
476 # test run we are only interested in coverage of this package.
477 # Use "python" instead of "." to remove test files from coverage.
478 options += " --cov=."
480 # Always enabled branch coverage and terminal summary
481 options += " --cov-branch --cov-report=term "
483 covfile = "{}-cov-{}.xml".format(prefix, self._env['eupsProduct'])
485 # We should specify the output directory explicitly unless the prefix
486 # indicates that we are using the SCons target
487 if covfile.startswith("${TARGET}"):
488 covpath = covfile
489 else:
490 covpath = os.path.join(self._tmpDirAbs, covfile)
491 options += " --cov-report=xml:'{}'".format(covpath)
493 # Use the prefix for the HTML output directory
494 htmlfile = ":'{}-htmlcov'".format(prefix)
495 options += " --cov-report=html{}".format(htmlfile)
497 return options