Coverage for python/lsst/sconsUtils/tests.py: 8%
171 statements
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:29 -0700
« prev ^ index » next coverage.py v7.5.0, created at 2024-04-24 23:29 -0700
1"""Control which tests run, and how.
2"""
4__all__ = ("Control", )
6import glob
7import os
8import sys
9import pipes
10import SCons.Script
11from . import state
12from . import utils
15class Control:
16 """A class to control and run unit tests.
18 This class is unchanged from previous versions of sconsUtils, but it will
19 now generally be called via
20 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
22 Parameters
23 ----------
24 env : `SCons.Environment`
25 An SCons Environment (almost always `lsst.sconsUtils.env`).
26 ignoreList : `list`, optional
27 A list of tests that should NOT be run --- useful in conjunction
28 with glob patterns. If a file is listed as "@fileName", the @ is
29 stripped and we don't bother to check if fileName exists (useful for
30 machine-generated files).
31 expectedFailures : `dict`, optional
32 A dictionary; the keys are tests that are known to fail; the values
33 are strings to print.
34 args : `dict`, optional
35 A dictionary with testnames as keys, and argument strings as values.
36 As scons always runs from the top-level directory, tests has to fiddle
37 with paths. If an argument is a file this is done automatically; if
38 it's e.g., just a basename then you have to tell tests that it's
39 really (part of a) filename by prefixing the name by ``file:``.
40 tmpDir : `str`, optional
41 The location of the test outputs.
42 verbose : `bool`, optional
43 How chatty you want the test code to be.
45 Notes
46 -----
47 Sample usage:
49 .. code-block:: python
51 tests = lsst.tests.Control(
52 env,
53 args={
54 "MaskIO_1" : "data/871034p_1_MI_msk.fits",
55 "MaskedImage_1" : "file:data/871034p_1_MI foo",
56 },
57 ignoreList=["Measure_1"],
58 expectedFailures={"BBox_1": "Problem with single-pixel BBox"}
59 )
61 This class is unchanged from previous versions of sconsUtils, but it will
62 now generally be called via
63 `lsst.sconsUtils.scripts.BasicSConscript.tests`.
64 """
66 _IGNORE = "IGNORE"
67 _EXPECT_FAILURE = "EXPECT_FAILURE"
69 def __init__(self, env, ignoreList=None, expectedFailures=None, args=None,
70 tmpDir=".tests", verbose=False):
72 # Need to define our own Astropy cache directories.
73 # Unfortunately we can not simply set XDG_CACHE_HOME
74 # to $HOME/.astropy. Do not forward $HOME or the XDG_CONFIG_HOME
75 # environment variables since those may affect test outcomes.
76 xdgCacheVar = "XDG_CACHE_HOME"
77 if xdgCacheVar not in os.environ:
78 if "~" in os.path.expanduser("~"):
79 state.log.warn(f"Neither $HOME nor ${xdgCacheVar} defined. No Astropy cache enabled.")
80 else:
81 # We need a directory for the cache and that directory
82 # has to have an "astropy" directory inside it. We can
83 # use ~/.astropy or ~/.lsst or a tmp directory but choose
84 # ~/.lsst initially.
85 cacheDir = os.path.expanduser("~/.lsst")
86 astropyCacheDir = os.path.join(cacheDir, "astropy")
87 if not os.path.exists(astropyCacheDir):
88 os.makedirs(astropyCacheDir, exist_ok=True) # Race condition is okay
89 os.environ[xdgCacheVar] = cacheDir
90 else:
91 if not os.path.exists(os.path.expanduser(os.path.join(os.environ[xdgCacheVar],
92 "astropy"))):
93 state.log.warn(f"{xdgCacheVar} is set but will not be used for "
94 "astropy due to lack of astropy directory within it")
96 # Forward some environment to the tests
97 for envvar in ["PYTHONPATH", xdgCacheVar]:
98 if envvar in os.environ:
99 env.AppendENVPath(envvar, os.environ[envvar])
101 self._env = env
103 self._tmpDir = tmpDir
104 self._cwd = os.path.abspath(os.path.curdir)
106 # Calculate the absolute path for temp dir if it is relative.
107 # This assumes the temp dir is relative to where the tests SConscript
108 # file is located. SCons will know how to handle this itself but
109 # some options require the code to know where to write things.
110 if os.path.isabs(self._tmpDir):
111 self._tmpDirAbs = self._tmpDir
112 else:
113 self._tmpDirAbs = os.path.join(self._cwd, self._tmpDir)
115 self._verbose = verbose
117 self._info = {} # information about processing targets
118 if ignoreList:
119 for f in ignoreList:
120 if f.startswith("@"): # @dfilename => don't complain if filename doesn't exist
121 f = f[1:]
122 else:
123 if not os.path.exists(f):
124 state.log.warn("You're ignoring a non-existent file, %s" % f)
125 self._info[f] = (self._IGNORE, None)
127 if expectedFailures:
128 for f in expectedFailures:
129 self._info[f] = (self._EXPECT_FAILURE, expectedFailures[f])
131 if args:
132 self._args = args # arguments for tests
133 else:
134 self._args = {}
136 self.runExamples = True # should I run the examples?
137 try:
138 # file is user read/write/executable
139 self.runExamples = (os.stat(self._tmpDir).st_mode & 0o700) != 0
140 except OSError:
141 pass
143 if not self.runExamples:
144 print("Not running examples; \"chmod 755 %s\" to run them again" % self._tmpDir,
145 file=sys.stderr)
147 def args(self, test):
148 """Arguments to use for this test.
150 Parameters
151 ----------
152 test : `str`
153 Test file to be run.
155 Returns
156 -------
157 args : `str`
158 The arguments as a single string. An empty string is returned
159 if no arguments were specified in the constructor.
160 """
161 try:
162 return self._args[test]
163 except KeyError:
164 return ""
166 def ignore(self, test):
167 """Should the test be ignored.
169 Parameters
170 ----------
171 test : `str`
172 The test target name.
174 Returns
175 -------
176 ignore : `bool`
177 Whether the test should be ignored or not.
178 """
179 if not test.endswith(".py") and \
180 len(self._env.Glob(test)) == 0: # we don't know how to build it
181 return True
183 ignoreFile = test in self._info and self._info[test][0] == self._IGNORE
185 if self._verbose and ignoreFile:
186 print("Skipping", test, file=sys.stderr)
188 return ignoreFile
190 def messages(self, test):
191 """Return the messages to be used in case of success/failure.
193 Parameters
194 ----------
195 test : `str`
196 The test target.
198 Returns
199 -------
200 messages : `tuple`
201 A `tuple` containing four strings: whether the test should pass
202 (as a value "true" or "false") and the associated message, and
203 whether the test should fail and the associated message.
204 """
206 if test in self._info and self._info[test][0] == self._EXPECT_FAILURE:
207 msg = self._info[test][1]
208 return ("false", "Passed, but should have failed: %s" % msg,
209 "true", "Failed as expected: %s" % msg)
210 else:
211 return ("true", "passed",
212 "false", "failed")
214 def run(self, fileGlob):
215 """Create a test target for each file matching the supplied glob.
217 Parameters
218 ----------
219 fileGlob : `str` or `SCons.Environment.Glob`
220 File matching glob.
222 Returns
223 -------
224 targets :
225 Test target for each matching file.
226 """
228 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
229 fileGlob = str(fileGlob)
230 targets = []
231 if not self.runExamples:
232 return targets
234 # Determine any library load path values that we have to prepend
235 # to the command.
236 libpathstr = utils.libraryLoaderEnvironment()
238 for f in glob.glob(fileGlob):
239 interpreter = "" # interpreter to run test, if needed
241 if f.endswith(".cc"): # look for executable
242 f = os.path.splitext(f)[0]
243 else:
244 interpreter = "pytest -Wd --durations=5 --junit-xml=${TARGET}.xml"
245 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
246 interpreter += self._getPytestCoverageCommand()
248 if self.ignore(f):
249 continue
251 target = os.path.join(self._tmpDir, f)
253 args = []
254 for a in self.args(f).split(" "):
255 # if a is a file, make it an absolute name as scons runs from
256 # the root directory
257 filePrefix = "file:"
258 if a.startswith(filePrefix): # they explicitly said that this was a file
259 a = os.path.join(self._cwd, a[len(filePrefix):])
260 else:
261 try: # see if it's a file
262 os.stat(a)
263 a = os.path.join(self._cwd, a)
264 except OSError:
265 pass
267 args += [a]
269 (should_pass, passedMsg, should_fail, failedMsg) = self.messages(f)
271 expandedArgs = " ".join(args)
272 result = self._env.Command(target, f, """
273 @rm -f ${TARGET}.failed;
274 @printf "%%s" 'running ${SOURCES}... ';
275 @echo $SOURCES %s > $TARGET; echo >> $TARGET;
276 @if %s %s $SOURCES %s >> $TARGET 2>&1; then \
277 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
278 echo "%s"; \
279 else \
280 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \
281 echo "%s"; \
282 fi;
283 """ % (expandedArgs, libpathstr, interpreter, expandedArgs, should_pass,
284 passedMsg, should_fail, failedMsg))
286 targets.extend(result)
288 self._env.Alias(os.path.basename(target), target)
290 self._env.Clean(target, self._tmpDir)
292 return targets
294 def runPythonTests(self, pyList):
295 """Add a single target for testing all python files.
297 Parameters
298 ----------
299 pyList : `list`
300 A list of nodes corresponding to python test files. The
301 IgnoreList is respected when scanning for entries. If pyList
302 is `None`, or an empty list, it uses automated test discovery
303 within pytest. This differs from the behavior of
304 `lsst.sconsUtils.BasicSconscript.tests`
305 where a distinction is made.
307 Returns
308 -------
309 target : `list`
310 Returns a list containing a single target.
311 """
313 if pyList is None:
314 pyList = []
316 # Determine any library load path values that we have to prepend
317 # to the command.
318 libpathstr = utils.libraryLoaderEnvironment()
320 # Get list of python files with the path included.
321 pythonTestFiles = []
322 for fileGlob in pyList:
323 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node
324 fileGlob = str(fileGlob)
325 for f in glob.glob(fileGlob):
326 if self.ignore(f):
327 continue
328 pythonTestFiles.append(os.path.join(self._cwd, f))
330 # Now set up the python testing target
331 # We always want to run this with the tests target.
332 # We have decided to use pytest caching so that on reruns we only
333 # run failed tests.
334 lfnfOpt = "none" if 'install' in SCons.Script.COMMAND_LINE_TARGETS else "all"
335 interpreter = f"pytest -Wd --lf --lfnf={lfnfOpt}"
336 interpreter += " --durations=5 --junit-xml=${TARGET} --session2file=${TARGET}.out"
337 interpreter += " --junit-prefix={0}".format(self.junitPrefix())
338 interpreter += self._getPytestCoverageCommand()
340 # Ignore doxygen build directories since they can confuse pytest
341 # test collection
342 interpreter += " --ignore=doc/html --ignore=doc/xml"
344 # Ignore the C++ directories since they will never have python
345 # code and doing this will speed up test collection
346 interpreter += " --ignore=src --ignore=include --ignore=lib"
348 # Ignore the eups directory
349 interpreter += " --ignore=ups"
351 # We currently have a race condition in test collection when
352 # examples has C++ code. Removing it from the scan will get us through
353 # until we can fix the problem properly. Rely on GitHub PRs to
354 # do the flake8 check.
355 interpreter += " --ignore=examples"
357 # Also include temporary files made by compilers.
358 # These can come from examples directories that include C++.
359 interpreter += " --ignore-glob='*.tmp'"
361 target = os.path.join(self._tmpDir, "pytest-{}.xml".format(self._env['eupsProduct']))
363 # Work out how many jobs scons has been configured to use
364 # and use that number with pytest. This could cause trouble
365 # if there are lots of binary tests to run and lots of singles.
366 njobs = self._env.GetOption("num_jobs")
367 print("Running pytest with {} process{}".format(njobs, "" if njobs == 1 else "es"))
368 if njobs > 1:
369 # We unambiguously specify the Python interpreter to be used to
370 # execute tests. This ensures that all pytest-xdist worker
371 # processes refer to the same Python as the xdist controller, and
372 # hence avoids pollution of ``sys.path`` that can happen when we
373 # call the same interpreter by different paths (for example, if
374 # the controller process calls ``miniconda/bin/python``, and the
375 # workers call ``current/bin/python``, the workers will end up
376 # with site-packages directories corresponding to both locations
377 # on ``sys.path``, even if the one is a symlink to the other).
378 executable = os.path.realpath(sys.executable)
380 # if there is a space in the executable path we have to use the
381 # original method and hope things work okay. This will be rare but
382 # without this a space in the path is impossible because of how
383 # xdist currently parses the tx option
384 interpreter = interpreter + " --max-worker-restart=0"
385 if " " not in executable:
386 interpreter = (interpreter
387 + " -d --tx={}*popen//python={}".format(njobs, executable))
388 else:
389 interpreter = interpreter + " -n {}".format(njobs)
391 # Remove target so that we always trigger pytest
392 if os.path.exists(target):
393 os.unlink(target)
395 if not pythonTestFiles:
396 print("pytest: automated test discovery mode enabled.")
397 else:
398 nfiles = len(pythonTestFiles)
399 print("pytest: running on {} Python test file{}.".format(nfiles, "" if nfiles == 1 else "s"))
401 # If we ran all the test, then copy the previous test
402 # execution products to `.all' files so we can retrieve later.
403 # If we skip the test (exit code 5), retrieve those `.all' files.
404 cmd = ""
405 if lfnfOpt == "all":
406 cmd += "@rm -f ${{TARGET}} ${{TARGET}}.failed;"
407 cmd += """
408 @printf "%s\\n" 'running global pytest... ';
409 @({2} {0} {1}); \
410 export rc="$?"; \
411 if [ "$$rc" -eq 0 ]; then \
412 echo "Global pytest run completed successfully"; \
413 cp ${{TARGET}} ${{TARGET}}.all || true; \
414 cp ${{TARGET}}.out ${{TARGET}}.out.all || true; \
415 elif [ "$$rc" -eq 5 ]; then \
416 echo "Global pytest run completed successfully - no tests ran"; \
417 mv ${{TARGET}}.all ${{TARGET}} || true; \
418 mv ${{TARGET}}.out.all ${{TARGET}}.out || true; \
419 else \
420 echo "Global pytest run: failed with $$rc"; \
421 mv ${{TARGET}}.out ${{TARGET}}.failed; \
422 fi;
423 """
424 testfiles = " ".join([pipes.quote(p) for p in pythonTestFiles])
425 result = self._env.Command(target, None, cmd.format(interpreter, testfiles, libpathstr))
427 self._env.Alias(os.path.basename(target), target)
428 self._env.Clean(target, self._tmpDir)
430 return [result]
432 def junitPrefix(self):
433 """Calculate the prefix to use for the JUnit output.
435 Returns
436 -------
437 prefix : `str`
438 Prefix string to use.
440 Notes
441 -----
442 Will use the EUPS product being built and the value of the
443 ``LSST_JUNIT_PREFIX`` environment variable if that is set.
444 """
445 controlVar = "LSST_JUNIT_PREFIX"
446 prefix = self._env['eupsProduct']
448 if controlVar in os.environ:
449 prefix += ".{0}".format(os.environ[controlVar])
451 return prefix
453 def _getPytestCoverageCommand(self):
454 """Form the additional arguments required to enable coverage testing.
456 Coverage output files are written using ``${TARGET}`` as a base.
458 Returns
459 -------
460 options : `str`
461 String defining the coverage-specific arguments to give to the
462 pytest command.
463 """
465 options = ""
467 # Basis for deriving file names
468 # We use the magic target from SCons.
469 prefix = "${TARGET}"
471 # Only report coverage for files in the build tree.
472 # If --cov is used full coverage will be reported for all installed
473 # code as well, but that is probably a distraction as for this
474 # test run we are only interested in coverage of this package.
475 # Use "python" instead of "." to remove test files from coverage.
476 options += " --cov=."
478 # Always enabled branch coverage and terminal summary
479 options += " --cov-branch --cov-report=term "
481 covfile = "{}-cov-{}.xml".format(prefix, self._env['eupsProduct'])
483 # We should specify the output directory explicitly unless the prefix
484 # indicates that we are using the SCons target
485 if covfile.startswith("${TARGET}"):
486 covpath = covfile
487 else:
488 covpath = os.path.join(self._tmpDirAbs, covfile)
489 options += " --cov-report=xml:'{}'".format(covpath)
491 # Use the prefix for the HTML output directory
492 htmlfile = ":'{}-htmlcov'".format(prefix)
493 options += " --cov-report=html{}".format(htmlfile)
495 return options