Coverage for python/lsst/sconsUtils/tests.py: 8%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

171 statements  

1"""Control which tests run, and how. 

2""" 

3 

4__all__ = ("Control", ) 

5 

6import glob 

7import os 

8import sys 

9import pipes 

10import SCons.Script 

11from . import state 

12from . import utils 

13 

14 

15class Control: 

16 """A class to control and run unit tests. 

17 

18 This class is unchanged from previous versions of sconsUtils, but it will 

19 now generally be called via 

20 `lsst.sconsUtils.scripts.BasicSConscript.tests`. 

21 

22 Parameters 

23 ---------- 

24 env : `SCons.Environment` 

25 An SCons Environment (almost always `lsst.sconsUtils.env`). 

26 ignoreList : `list`, optional 

27 A list of tests that should NOT be run --- useful in conjunction 

28 with glob patterns. If a file is listed as "@fileName", the @ is 

29 stripped and we don't bother to check if fileName exists (useful for 

30 machine-generated files). 

31 expectedFailures : `dict`, optional 

32 A dictionary; the keys are tests that are known to fail; the values 

33 are strings to print. 

34 args : `dict`, optional 

35 A dictionary with testnames as keys, and argument strings as values. 

36 As scons always runs from the top-level directory, tests has to fiddle 

37 with paths. If an argument is a file this is done automatically; if 

38 it's e.g., just a basename then you have to tell tests that it's 

39 really (part of a) filename by prefixing the name by ``file:``. 

40 tmpDir : `str`, optional 

41 The location of the test outputs. 

42 verbose : `bool`, optional 

43 How chatty you want the test code to be. 

44 

45 Notes 

46 ----- 

47 Sample usage: 

48 

49 .. code-block:: python 

50 

51 tests = lsst.tests.Control( 

52 env, 

53 args={ 

54 "MaskIO_1" : "data/871034p_1_MI_msk.fits", 

55 "MaskedImage_1" : "file:data/871034p_1_MI foo", 

56 }, 

57 ignoreList=["Measure_1"], 

58 expectedFailures={"BBox_1": "Problem with single-pixel BBox"} 

59 ) 

60 

61 This class is unchanged from previous versions of sconsUtils, but it will 

62 now generally be called via 

63 `lsst.sconsUtils.scripts.BasicSConscript.tests`. 

64 """ 

65 

66 _IGNORE = "IGNORE" 

67 _EXPECT_FAILURE = "EXPECT_FAILURE" 

68 

69 def __init__(self, env, ignoreList=None, expectedFailures=None, args=None, 

70 tmpDir=".tests", verbose=False): 

71 

72 # Need to define our own Astropy cache directories. 

73 # Unfortunately we can not simply set XDG_CACHE_HOME 

74 # to $HOME/.astropy. Do not forward $HOME or the XDG_CONFIG_HOME 

75 # environment variables since those may affect test outcomes. 

76 xdgCacheVar = "XDG_CACHE_HOME" 

77 if xdgCacheVar not in os.environ: 

78 if "~" in os.path.expanduser("~"): 

79 state.log.warn(f"Neither $HOME nor ${xdgCacheVar} defined. No Astropy cache enabled.") 

80 else: 

81 # We need a directory for the cache and that directory 

82 # has to have an "astropy" directory inside it. We can 

83 # use ~/.astropy or ~/.lsst or a tmp directory but choose 

84 # ~/.lsst initially. 

85 cacheDir = os.path.expanduser("~/.lsst") 

86 astropyCacheDir = os.path.join(cacheDir, "astropy") 

87 if not os.path.exists(astropyCacheDir): 

88 os.makedirs(astropyCacheDir, exist_ok=True) # Race condition is okay 

89 os.environ[xdgCacheVar] = cacheDir 

90 else: 

91 if not os.path.exists(os.path.expanduser(os.path.join(os.environ[xdgCacheVar], 

92 "astropy"))): 

93 state.log.warn(f"{xdgCacheVar} is set but will not be used for " 

94 "astropy due to lack of astropy directory within it") 

95 

96 # Forward some environment to the tests 

97 for envvar in ["PYTHONPATH", xdgCacheVar]: 

98 if envvar in os.environ: 

99 env.AppendENVPath(envvar, os.environ[envvar]) 

100 

101 self._env = env 

102 

103 self._tmpDir = tmpDir 

104 self._cwd = os.path.abspath(os.path.curdir) 

105 

106 # Calculate the absolute path for temp dir if it is relative. 

107 # This assumes the temp dir is relative to where the tests SConscript 

108 # file is located. SCons will know how to handle this itself but 

109 # some options require the code to know where to write things. 

110 if os.path.isabs(self._tmpDir): 

111 self._tmpDirAbs = self._tmpDir 

112 else: 

113 self._tmpDirAbs = os.path.join(self._cwd, self._tmpDir) 

114 

115 self._verbose = verbose 

116 

117 self._info = {} # information about processing targets 

118 if ignoreList: 

119 for f in ignoreList: 

120 if f.startswith("@"): # @dfilename => don't complain if filename doesn't exist 

121 f = f[1:] 

122 else: 

123 if not os.path.exists(f): 

124 state.log.warn("You're ignoring a non-existent file, %s" % f) 

125 self._info[f] = (self._IGNORE, None) 

126 

127 if expectedFailures: 

128 for f in expectedFailures: 

129 self._info[f] = (self._EXPECT_FAILURE, expectedFailures[f]) 

130 

131 if args: 

132 self._args = args # arguments for tests 

133 else: 

134 self._args = {} 

135 

136 self.runExamples = True # should I run the examples? 

137 try: 

138 # file is user read/write/executable 

139 self.runExamples = (os.stat(self._tmpDir).st_mode & 0o700) != 0 

140 except OSError: 

141 pass 

142 

143 if not self.runExamples: 

144 print("Not running examples; \"chmod 755 %s\" to run them again" % self._tmpDir, 

145 file=sys.stderr) 

146 

147 def args(self, test): 

148 """Arguments to use for this test. 

149 

150 Parameters 

151 ---------- 

152 test : `str` 

153 Test file to be run. 

154 

155 Returns 

156 ------- 

157 args : `str` 

158 The arguments as a single string. An empty string is returned 

159 if no arguments were specified in the constructor. 

160 """ 

161 try: 

162 return self._args[test] 

163 except KeyError: 

164 return "" 

165 

166 def ignore(self, test): 

167 """Should the test be ignored. 

168 

169 Parameters 

170 ---------- 

171 test : `str` 

172 The test target name. 

173 

174 Returns 

175 ------- 

176 ignore : `bool` 

177 Whether the test should be ignored or not. 

178 """ 

179 if not test.endswith(".py") and \ 

180 len(self._env.Glob(test)) == 0: # we don't know how to build it 

181 return True 

182 

183 ignoreFile = test in self._info and self._info[test][0] == self._IGNORE 

184 

185 if self._verbose and ignoreFile: 

186 print("Skipping", test, file=sys.stderr) 

187 

188 return ignoreFile 

189 

190 def messages(self, test): 

191 """Return the messages to be used in case of success/failure. 

192 

193 Parameters 

194 ---------- 

195 test : `str` 

196 The test target. 

197 

198 Returns 

199 ------- 

200 messages : `tuple` 

201 A `tuple` containing four strings: whether the test should pass 

202 (as a value "true" or "false") and the associated message, and 

203 whether the test should fail and the associated message. 

204 """ 

205 

206 if test in self._info and self._info[test][0] == self._EXPECT_FAILURE: 

207 msg = self._info[test][1] 

208 return ("false", "Passed, but should have failed: %s" % msg, 

209 "true", "Failed as expected: %s" % msg) 

210 else: 

211 return ("true", "passed", 

212 "false", "failed") 

213 

214 def run(self, fileGlob): 

215 """Create a test target for each file matching the supplied glob. 

216 

217 Parameters 

218 ---------- 

219 fileGlob : `str` or `SCons.Environment.Glob` 

220 File matching glob. 

221 

222 Returns 

223 ------- 

224 targets : 

225 Test target for each matching file. 

226 """ 

227 

228 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node 

229 fileGlob = str(fileGlob) 

230 targets = [] 

231 if not self.runExamples: 

232 return targets 

233 

234 # Determine any library load path values that we have to prepend 

235 # to the command. 

236 libpathstr = utils.libraryLoaderEnvironment() 

237 

238 for f in glob.glob(fileGlob): 

239 interpreter = "" # interpreter to run test, if needed 

240 

241 if f.endswith(".cc"): # look for executable 

242 f = os.path.splitext(f)[0] 

243 else: 

244 interpreter = "pytest -Wd --durations=5 --junit-xml=${TARGET}.xml" 

245 interpreter += " --junit-prefix={0}".format(self.junitPrefix()) 

246 interpreter += self._getPytestCoverageCommand() 

247 

248 if self.ignore(f): 

249 continue 

250 

251 target = os.path.join(self._tmpDir, f) 

252 

253 args = [] 

254 for a in self.args(f).split(" "): 

255 # if a is a file, make it an absolute name as scons runs from 

256 # the root directory 

257 filePrefix = "file:" 

258 if a.startswith(filePrefix): # they explicitly said that this was a file 

259 a = os.path.join(self._cwd, a[len(filePrefix):]) 

260 else: 

261 try: # see if it's a file 

262 os.stat(a) 

263 a = os.path.join(self._cwd, a) 

264 except OSError: 

265 pass 

266 

267 args += [a] 

268 

269 (should_pass, passedMsg, should_fail, failedMsg) = self.messages(f) 

270 

271 expandedArgs = " ".join(args) 

272 result = self._env.Command(target, f, """ 

273 @rm -f ${TARGET}.failed; 

274 @printf "%%s" 'running ${SOURCES}... '; 

275 @echo $SOURCES %s > $TARGET; echo >> $TARGET; 

276 @if %s %s $SOURCES %s >> $TARGET 2>&1; then \ 

277 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \ 

278 echo "%s"; \ 

279 else \ 

280 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \ 

281 echo "%s"; \ 

282 fi; 

283 """ % (expandedArgs, libpathstr, interpreter, expandedArgs, should_pass, 

284 passedMsg, should_fail, failedMsg)) 

285 

286 targets.extend(result) 

287 

288 self._env.Alias(os.path.basename(target), target) 

289 

290 self._env.Clean(target, self._tmpDir) 

291 

292 return targets 

293 

294 def runPythonTests(self, pyList): 

295 """Add a single target for testing all python files. 

296 

297 Parameters 

298 ---------- 

299 pyList : `list` 

300 A list of nodes corresponding to python test files. The 

301 IgnoreList is respected when scanning for entries. If pyList 

302 is `None`, or an empty list, it uses automated test discovery 

303 within pytest. This differs from the behavior of 

304 `lsst.sconsUtils.BasicSconscript.tests` 

305 where a distinction is made. 

306 

307 Returns 

308 ------- 

309 target : `list` 

310 Returns a list containing a single target. 

311 """ 

312 

313 if pyList is None: 

314 pyList = [] 

315 

316 # Determine any library load path values that we have to prepend 

317 # to the command. 

318 libpathstr = utils.libraryLoaderEnvironment() 

319 

320 # Get list of python files with the path included. 

321 pythonTestFiles = [] 

322 for fileGlob in pyList: 

323 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node 

324 fileGlob = str(fileGlob) 

325 for f in glob.glob(fileGlob): 

326 if self.ignore(f): 

327 continue 

328 pythonTestFiles.append(os.path.join(self._cwd, f)) 

329 

330 # Now set up the python testing target 

331 # We always want to run this with the tests target. 

332 # We have decided to use pytest caching so that on reruns we only 

333 # run failed tests. 

334 lfnfOpt = "none" if 'install' in SCons.Script.COMMAND_LINE_TARGETS else "all" 

335 interpreter = f"pytest -Wd --lf --lfnf={lfnfOpt}" 

336 interpreter += " --durations=5 --junit-xml=${TARGET} --session2file=${TARGET}.out" 

337 interpreter += " --junit-prefix={0}".format(self.junitPrefix()) 

338 interpreter += self._getPytestCoverageCommand() 

339 

340 # Ignore doxygen build directories since they can confuse pytest 

341 # test collection 

342 interpreter += " --ignore=doc/html --ignore=doc/xml" 

343 

344 # Ignore the C++ directories since they will never have python 

345 # code and doing this will speed up test collection 

346 interpreter += " --ignore=src --ignore=include --ignore=lib" 

347 

348 # Ignore the eups directory 

349 interpreter += " --ignore=ups" 

350 

351 # We currently have a race condition in test collection when 

352 # examples has C++ code. Removing it from the scan will get us through 

353 # until we can fix the problem properly. Rely on GitHub PRs to 

354 # do the flake8 check. 

355 interpreter += " --ignore=examples" 

356 

357 # Also include temporary files made by compilers. 

358 # These can come from examples directories that include C++. 

359 interpreter += " --ignore-glob='*.tmp'" 

360 

361 target = os.path.join(self._tmpDir, "pytest-{}.xml".format(self._env['eupsProduct'])) 

362 

363 # Work out how many jobs scons has been configured to use 

364 # and use that number with pytest. This could cause trouble 

365 # if there are lots of binary tests to run and lots of singles. 

366 njobs = self._env.GetOption("num_jobs") 

367 print("Running pytest with {} process{}".format(njobs, "" if njobs == 1 else "es")) 

368 if njobs > 1: 

369 # We unambiguously specify the Python interpreter to be used to 

370 # execute tests. This ensures that all pytest-xdist worker 

371 # processes refer to the same Python as the xdist controller, and 

372 # hence avoids pollution of ``sys.path`` that can happen when we 

373 # call the same interpreter by different paths (for example, if 

374 # the controller process calls ``miniconda/bin/python``, and the 

375 # workers call ``current/bin/python``, the workers will end up 

376 # with site-packages directories corresponding to both locations 

377 # on ``sys.path``, even if the one is a symlink to the other). 

378 executable = os.path.realpath(sys.executable) 

379 

380 # if there is a space in the executable path we have to use the 

381 # original method and hope things work okay. This will be rare but 

382 # without this a space in the path is impossible because of how 

383 # xdist currently parses the tx option 

384 interpreter = interpreter + " --max-worker-restart=0" 

385 if " " not in executable: 

386 interpreter = (interpreter 

387 + " -d --tx={}*popen//python={}".format(njobs, executable)) 

388 else: 

389 interpreter = interpreter + " -n {}".format(njobs) 

390 

391 # Remove target so that we always trigger pytest 

392 if os.path.exists(target): 

393 os.unlink(target) 

394 

395 if not pythonTestFiles: 

396 print("pytest: automated test discovery mode enabled.") 

397 else: 

398 nfiles = len(pythonTestFiles) 

399 print("pytest: running on {} Python test file{}.".format(nfiles, "" if nfiles == 1 else "s")) 

400 

401 # If we ran all the test, then copy the previous test 

402 # execution products to `.all' files so we can retrieve later. 

403 # If we skip the test (exit code 5), retrieve those `.all' files. 

404 cmd = "" 

405 if lfnfOpt == "all": 

406 cmd += "@rm -f ${{TARGET}} ${{TARGET}}.failed;" 

407 cmd += """ 

408 @printf "%s\\n" 'running global pytest... '; 

409 @({2} {0} {1}); \ 

410 export rc="$?"; \ 

411 if [ "$$rc" -eq 0 ]; then \ 

412 echo "Global pytest run completed successfully"; \ 

413 cp ${{TARGET}} ${{TARGET}}.all || true; \ 

414 cp ${{TARGET}}.out ${{TARGET}}.out.all || true; \ 

415 elif [ "$$rc" -eq 5 ]; then \ 

416 echo "Global pytest run completed successfully - no tests ran"; \ 

417 mv ${{TARGET}}.all ${{TARGET}} || true; \ 

418 mv ${{TARGET}}.out.all ${{TARGET}}.out || true; \ 

419 else \ 

420 echo "Global pytest run: failed with $$rc"; \ 

421 mv ${{TARGET}}.out ${{TARGET}}.failed; \ 

422 fi; 

423 """ 

424 testfiles = " ".join([pipes.quote(p) for p in pythonTestFiles]) 

425 result = self._env.Command(target, None, cmd.format(interpreter, testfiles, libpathstr)) 

426 

427 self._env.Alias(os.path.basename(target), target) 

428 self._env.Clean(target, self._tmpDir) 

429 

430 return [result] 

431 

432 def junitPrefix(self): 

433 """Calculate the prefix to use for the JUnit output. 

434 

435 Returns 

436 ------- 

437 prefix : `str` 

438 Prefix string to use. 

439 

440 Notes 

441 ----- 

442 Will use the EUPS product being built and the value of the 

443 ``LSST_JUNIT_PREFIX`` environment variable if that is set. 

444 """ 

445 controlVar = "LSST_JUNIT_PREFIX" 

446 prefix = self._env['eupsProduct'] 

447 

448 if controlVar in os.environ: 

449 prefix += ".{0}".format(os.environ[controlVar]) 

450 

451 return prefix 

452 

453 def _getPytestCoverageCommand(self): 

454 """Form the additional arguments required to enable coverage testing. 

455 

456 Coverage output files are written using ``${TARGET}`` as a base. 

457 

458 Returns 

459 ------- 

460 options : `str` 

461 String defining the coverage-specific arguments to give to the 

462 pytest command. 

463 """ 

464 

465 options = "" 

466 

467 # Basis for deriving file names 

468 # We use the magic target from SCons. 

469 prefix = "${TARGET}" 

470 

471 # Only report coverage for files in the build tree. 

472 # If --cov is used full coverage will be reported for all installed 

473 # code as well, but that is probably a distraction as for this 

474 # test run we are only interested in coverage of this package. 

475 # Use "python" instead of "." to remove test files from coverage. 

476 options += " --cov=." 

477 

478 # Always enabled branch coverage and terminal summary 

479 options += " --cov-branch --cov-report=term " 

480 

481 covfile = "{}-cov-{}.xml".format(prefix, self._env['eupsProduct']) 

482 

483 # We should specify the output directory explicitly unless the prefix 

484 # indicates that we are using the SCons target 

485 if covfile.startswith("${TARGET}"): 

486 covpath = covfile 

487 else: 

488 covpath = os.path.join(self._tmpDirAbs, covfile) 

489 options += " --cov-report=xml:'{}'".format(covpath) 

490 

491 # Use the prefix for the HTML output directory 

492 htmlfile = ":'{}-htmlcov'".format(prefix) 

493 options += " --cov-report=html{}".format(htmlfile) 

494 

495 return options