Coverage for python/lsst/sconsUtils/tests.py: 8%
175 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-19 10:26 +0000
« prev ^ index » next coverage.py v7.2.3, created at 2023-04-19 10:26 +0000
1"""Control which tests run, and how.
2"""
4__all__ = ("Control", )
6import glob
7import os
8import sys
9import pipes
10import SCons.Script
11from . import state
12from . import utils
15class Control:
16 """A class to control and run unit tests.
18 This class is unchanged from previous versions of sconsUtils, but it will
19 now generally be called via
20 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
22 Parameters
23 ----------
24 env : `SCons.Environment`
25 An SCons Environment (almost always `lsst.sconsUtils.env`).
26 ignoreList : `list`, optional
27 A list of tests that should NOT be run --- useful in conjunction
28 with glob patterns. If a file is listed as "@fileName", the @ is
29 stripped and we don't bother to check if fileName exists (useful for
30 machine-generated files).
31 expectedFailures : `dict`, optional
32 A dictionary; the keys are tests that are known to fail; the values
33 are strings to print.
34 args : `dict`, optional
35 A dictionary with testnames as keys, and argument strings as values.
36 As scons always runs from the top-level directory, tests has to fiddle
37 with paths. If an argument is a file this is done automatically; if
38 it's e.g., just a basename then you have to tell tests that it's
39 really (part of a) filename by prefixing the name by ``file:``.
40 tmpDir : `str`, optional
41 The location of the test outputs.
42 verbose : `bool`, optional
43 How chatty you want the test code to be.
45 Notes
46 -----
47 Sample usage:
49 .. code-block:: python
51 tests = lsst.tests.Control(
52 env,
53 args={
54 "MaskIO_1" : "data/871034p_1_MI_msk.fits",
55 "MaskedImage_1" : "file:data/871034p_1_MI foo",
56 },
57 ignoreList=["Measure_1"],
58 expectedFailures={"BBox_1": "Problem with single-pixel BBox"}
59 )
61 This class is unchanged from previous versions of sconsUtils, but it will
62 now generally be called via
63 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
64 """
66 _IGNORE = "IGNORE"
67 _EXPECT_FAILURE = "EXPECT_FAILURE"
69 def __init__(self, env, ignoreList=None, expectedFailures=None, args=None,
70 tmpDir=".tests", verbose=False):
72 # Need to define our own Astropy cache directories.
73 # Unfortunately we can not simply set XDG_CACHE_HOME
74 # to $HOME/.astropy. Do not forward $HOME or the XDG_CONFIG_HOME
75 # environment variables since those may affect test outcomes.
76 xdgCacheVar = "XDG_CACHE_HOME"
77 if xdgCacheVar not in os.environ:
78 if "~" in os.path.expanduser("~"):
79 state.log.warn(f"Neither $HOME nor ${xdgCacheVar} defined. No Astropy cache enabled.")
80 else:
81 # We need a directory for the cache and that directory
82 # has to have an "astropy" directory inside it. We can
83 # use ~/.astropy or ~/.lsst or a tmp directory but choose
84 # ~/.lsst initially.
85 cacheDir = os.path.expanduser("~/.lsst")
86 astropyCacheDir = os.path.join(cacheDir, "astropy")
87 if not os.path.exists(astropyCacheDir):
88 os.makedirs(astropyCacheDir, exist_ok=True) # Race condition is okay
89 os.environ[xdgCacheVar] = cacheDir
90 else:
91 if not os.path.exists(os.path.expanduser(os.path.join(os.environ[xdgCacheVar],
92 "astropy"))):
93 state.log.warn(f"{xdgCacheVar} is set but will not be used for "
94 "astropy due to lack of astropy directory within it")
96 # Forward some environment to the tests
97 for envvar in ["PYTHONPATH", "HTTP_PROXY", "HTTPS_PROXY", xdgCacheVar]:
98 if envvar in os.environ:
99 env.AppendENVPath(envvar, os.environ[envvar])
101 # We can't use the AppendENVPath call above as that mangles anything
102 # with ':' separators.
103 if "PYTHONWARNINGS" in os.environ:
104 env["ENV"]["PYTHONWARNINGS"] = os.environ["PYTHONWARNINGS"]
106 self._env = env
108 self._tmpDir = tmpDir
109 self._cwd = os.path.abspath(os.path.curdir)
111 # Calculate the absolute path for temp dir if it is relative.
112 # This assumes the temp dir is relative to where the tests SConscript
113 # file is located. SCons will know how to handle this itself but
114 # some options require the code to know where to write things.
115 if os.path.isabs(self._tmpDir):
116 self._tmpDirAbs = self._tmpDir
117 else:
118 self._tmpDirAbs = os.path.join(self._cwd, self._tmpDir)
120 self._verbose = verbose
122 self._info = {} # information about processing targets
123 if ignoreList:
124 for f in ignoreList:
125 if f.startswith("@"): # @dfilename => don't complain if filename doesn't exist
126 f = f[1:]
127 else:
128 if not os.path.exists(f):
129 state.log.warn("You're ignoring a non-existent file, %s" % f)
130 self._info[f] = (self._IGNORE, None)
132 if expectedFailures:
133 for f in expectedFailures:
134 self._info[f] = (self._EXPECT_FAILURE, expectedFailures[f])
136 if args:
137 self._args = args # arguments for tests
138 else:
139 self._args = {}
141 self.runExamples = True # should I run the examples?
142 try:
143 # file is user read/write/executable
144 self.runExamples = (os.stat(self._tmpDir).st_mode & 0o700) != 0
145 except OSError:
146 pass
148 if not self.runExamples:
149 print("Not running examples; \"chmod 755 %s\" to run them again" % self._tmpDir,
150 file=sys.stderr)
152 def args(self, test):
153 """Arguments to use for this test.
155 Parameters
156 ----------
157 test : `str`
158 Test file to be run.
160 Returns
161 -------
162 args : `str`
163 The arguments as a single string. An empty string is returned
164 if no arguments were specified in the constructor.
165 """
166 try:
167 return self._args[test]
168 except KeyError:
169 return ""
171 def ignore(self, test):
172 """Should the test be ignored.
174 Parameters
175 ----------
176 test : `str`
177 The test target name.
179 Returns
180 -------
181 ignore : `bool`
182 Whether the test should be ignored or not.
183 """
184 if not test.endswith(".py") and \
185 len(self._env.Glob(test)) == 0: # we don't know how to build it
186 return True
188 ignoreFile = test in self._info and self._info[test][0] == self._IGNORE
190 if self._verbose and ignoreFile:
191 print("Skipping", test, file=sys.stderr)
193 return ignoreFile
195 def messages(self, test):
196 """Return the messages to be used in case of success/failure.
198 Parameters
199 ----------
200 test : `str`
201 The test target.
203 Returns
204 -------
205 messages : `tuple`
206 A `tuple` containing four strings: whether the test should pass
207 (as a value "true" or "false") and the associated message, and
208 whether the test should fail and the associated message.
209 """
211 if test in self._info and self._info[test][0] == self._EXPECT_FAILURE:
212 msg = self._info[test][1]
213 return ("false", "Passed, but should have failed: %s" % msg,
214 "true", "Failed as expected: %s" % msg)
215 else:
216 return ("true", "passed",
217 "false", "failed")
219 def run(self, fileGlob):
220 """Create a test target for each file matching the supplied glob.
222 Parameters
223 ----------
224 fileGlob : `str` or `SCons.Environment.Glob`
225 File matching glob.
227 Returns
228 -------
229 targets :
230 Test target for each matching file.
231 """
233 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
234 fileGlob = str(fileGlob)
235 targets = []
236 if not self.runExamples:
237 return targets
239 # Determine any library load path values that we have to prepend
240 # to the command.
241 libpathstr = utils.libraryLoaderEnvironment()
243 for f in glob.glob(fileGlob):
244 interpreter = "" # interpreter to run test, if needed
246 if f.endswith(".cc"): # look for executable
247 f = os.path.splitext(f)[0]
248 else:
249 interpreter = "pytest -Wd --durations=5 --junit-xml=${TARGET}.xml"
250 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
251 interpreter += " --log-level=DEBUG"
252 interpreter += self._getPytestCoverageCommand()
254 if self.ignore(f):
255 continue
257 target = os.path.join(self._tmpDir, f)
259 args = []
260 for a in self.args(f).split(" "):
261 # if a is a file, make it an absolute name as scons runs from
262 # the root directory
263 filePrefix = "file:"
264 if a.startswith(filePrefix): # they explicitly said that this was a file
265 a = os.path.join(self._cwd, a[len(filePrefix):])
266 else:
267 try: # see if it's a file
268 os.stat(a)
269 a = os.path.join(self._cwd, a)
270 except OSError:
271 pass
273 args += [a]
275 (should_pass, passedMsg, should_fail, failedMsg) = self.messages(f)
277 expandedArgs = " ".join(args)
278 result = self._env.Command(target, f, """
279 @rm -f ${TARGET}.failed;
280 @printf "%%s" 'running ${SOURCES}... ';
281 @echo $SOURCES %s > $TARGET; echo >> $TARGET;
282 @if %s %s $SOURCES %s >> $TARGET 2>&1; then \
283 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
284 echo "%s"; \
285 else \
286 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
287 echo "%s"; \
288 fi;
289 """ % (expandedArgs, libpathstr, interpreter, expandedArgs, should_pass,
290 passedMsg, should_fail, failedMsg))
292 targets.extend(result)
294 self._env.Alias(os.path.basename(target), target)
296 self._env.Clean(target, self._tmpDir)
298 return targets
300 def runPythonTests(self, pyList):
301 """Add a single target for testing all python files.
303 Parameters
304 ----------
305 pyList : `list`
306 A list of nodes corresponding to python test files. The
307 IgnoreList is respected when scanning for entries. If pyList
308 is `None`, or an empty list, it uses automated test discovery
309 within pytest. This differs from the behavior of
310 `lsst.sconsUtils.BasicSconscript.tests`
311 where a distinction is made.
313 Returns
314 -------
315 target : `list`
316 Returns a list containing a single target.
317 """
319 if pyList is None:
320 pyList = []
322 # Determine any library load path values that we have to prepend
323 # to the command.
324 libpathstr = utils.libraryLoaderEnvironment()
326 # Get list of python files with the path included.
327 pythonTestFiles = []
328 for fileGlob in pyList:
329 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
330 fileGlob = str(fileGlob)
331 for f in glob.glob(fileGlob):
332 if self.ignore(f):
333 continue
334 pythonTestFiles.append(os.path.join(self._cwd, f))
336 # Now set up the python testing target
337 # We always want to run this with the tests target.
338 # We have decided to use pytest caching so that on reruns we only
339 # run failed tests.
340 lfnfOpt = "none" if 'install' in SCons.Script.COMMAND_LINE_TARGETS else "all"
341 interpreter = f"pytest -Wd --lf --lfnf={lfnfOpt}"
342 interpreter += " --durations=5 --junit-xml=${TARGET} --session2file=${TARGET}.out"
343 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
344 interpreter += " --log-level=DEBUG"
345 interpreter += self._getPytestCoverageCommand()
347 # Ignore doxygen build directories since they can confuse pytest
348 # test collection
349 interpreter += " --ignore=doc/html --ignore=doc/xml"
351 # Ignore the C++ directories since they will never have python
352 # code and doing this will speed up test collection
353 interpreter += " --ignore=src --ignore=include --ignore=lib"
355 # Ignore the eups directory
356 interpreter += " --ignore=ups"
358 # We currently have a race condition in test collection when
359 # examples has C++ code. Removing it from the scan will get us through
360 # until we can fix the problem properly. Rely on GitHub PRs to
361 # do the flake8 check.
362 interpreter += " --ignore=examples"
364 # Also include temporary files made by compilers.
365 # These can come from examples directories that include C++.
366 interpreter += " --ignore-glob='*.tmp'"
368 target = os.path.join(self._tmpDir, "pytest-{}.xml".format(self._env['eupsProduct']))
370 # Work out how many jobs scons has been configured to use
371 # and use that number with pytest. This could cause trouble
372 # if there are lots of binary tests to run and lots of singles.
373 njobs = self._env.GetOption("num_jobs")
374 print("Running pytest with {} process{}".format(njobs, "" if njobs == 1 else "es"))
375 if njobs > 1:
376 # We unambiguously specify the Python interpreter to be used to
377 # execute tests. This ensures that all pytest-xdist worker
378 # processes refer to the same Python as the xdist controller, and
379 # hence avoids pollution of ``sys.path`` that can happen when we
380 # call the same interpreter by different paths (for example, if
381 # the controller process calls ``miniconda/bin/python``, and the
382 # workers call ``current/bin/python``, the workers will end up
383 # with site-packages directories corresponding to both locations
384 # on ``sys.path``, even if the one is a symlink to the other).
385 executable = os.path.realpath(sys.executable)
387 # if there is a space in the executable path we have to use the
388 # original method and hope things work okay. This will be rare but
389 # without this a space in the path is impossible because of how
390 # xdist currently parses the tx option
391 interpreter = interpreter + " --max-worker-restart=0"
392 if " " not in executable:
393 interpreter = (interpreter
394 + " -d --tx={}*popen//python={}".format(njobs, executable))
395 else:
396 interpreter = interpreter + " -n {}".format(njobs)
398 # Remove target so that we always trigger pytest
399 if os.path.exists(target):
400 os.unlink(target)
402 if not pythonTestFiles:
403 print("pytest: automated test discovery mode enabled.")
404 else:
405 nfiles = len(pythonTestFiles)
406 print("pytest: running on {} Python test file{}.".format(nfiles, "" if nfiles == 1 else "s"))
408 # If we ran all the test, then copy the previous test
409 # execution products to `.all' files so we can retrieve later.
410 # If we skip the test (exit code 5), retrieve those `.all' files.
411 cmd = ""
412 if lfnfOpt == "all":
413 cmd += "@rm -f ${{TARGET}} ${{TARGET}}.failed;"
414 cmd += """
415 @printf "%s\\n" 'running global pytest... ';
416 @({2} {0} {1}); \
417 export rc="$?"; \
418 if [ "$$rc" -eq 0 ]; then \
419 echo "Global pytest run completed successfully"; \
420 cp ${{TARGET}} ${{TARGET}}.all || true; \
421 cp ${{TARGET}}.out ${{TARGET}}.out.all || true; \
422 elif [ "$$rc" -eq 5 ]; then \
423 echo "Global pytest run completed successfully - no tests ran"; \
424 mv ${{TARGET}}.all ${{TARGET}} || true; \
425 mv ${{TARGET}}.out.all ${{TARGET}}.out || true; \
426 else \
427 echo "Global pytest run: failed with $$rc"; \
428 mv ${{TARGET}}.out ${{TARGET}}.failed; \
429 fi;
430 """
431 testfiles = " ".join([pipes.quote(p) for p in pythonTestFiles])
432 result = self._env.Command(target, None, cmd.format(interpreter, testfiles, libpathstr))
434 self._env.Alias(os.path.basename(target), target)
435 self._env.Clean(target, self._tmpDir)
437 return [result]
439 def junitPrefix(self):
440 """Calculate the prefix to use for the JUnit output.
442 Returns
443 -------
444 prefix : `str`
445 Prefix string to use.
447 Notes
448 -----
449 Will use the EUPS product being built and the value of the
450 ``LSST_JUNIT_PREFIX`` environment variable if that is set.
451 """
452 controlVar = "LSST_JUNIT_PREFIX"
453 prefix = self._env['eupsProduct']
455 if controlVar in os.environ:
456 prefix += ".{0}".format(os.environ[controlVar])
458 return prefix
460 def _getPytestCoverageCommand(self):
461 """Form the additional arguments required to enable coverage testing.
463 Coverage output files are written using ``${TARGET}`` as a base.
465 Returns
466 -------
467 options : `str`
468 String defining the coverage-specific arguments to give to the
469 pytest command.
470 """
472 options = ""
474 # Basis for deriving file names
475 # We use the magic target from SCons.
476 prefix = "${TARGET}"
478 # Only report coverage for files in the build tree.
479 # If --cov is used full coverage will be reported for all installed
480 # code as well, but that is probably a distraction as for this
481 # test run we are only interested in coverage of this package.
482 # Use "python" instead of "." to remove test files from coverage.
483 options += " --cov=."
485 # Always enabled branch coverage and terminal summary
486 options += " --cov-branch --cov-report=term "
488 covfile = "{}-cov-{}.xml".format(prefix, self._env['eupsProduct'])
490 # We should specify the output directory explicitly unless the prefix
491 # indicates that we are using the SCons target
492 if covfile.startswith("${TARGET}"):
493 covpath = covfile
494 else:
495 covpath = os.path.join(self._tmpDirAbs, covfile)
496 options += " --cov-report=xml:'{}'".format(covpath)
498 # Use the prefix for the HTML output directory
499 htmlfile = ":'{}-htmlcov'".format(prefix)
500 options += " --cov-report=html{}".format(htmlfile)
502 return options