Coverage for python/lsst/sconsUtils/tests.py: 8%

175 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-19 10:26 +0000

1"""Control which tests run, and how. 

2""" 

3 

4__all__ = ("Control", ) 

5 

6import glob 

7import os 

8import sys 

9import pipes 

10import SCons.Script 

11from . import state 

12from . import utils 

13 

14 

15class Control: 

16 """A class to control and run unit tests. 

17 

18 This class is unchanged from previous versions of sconsUtils, but it will 

19 now generally be called via 

20 `lsst.sconsUtils.scripts.BasicSConscript.tests`. 

21 

22 Parameters 

23 ---------- 

24 env : `SCons.Environment` 

25 An SCons Environment (almost always `lsst.sconsUtils.env`). 

26 ignoreList : `list`, optional 

27 A list of tests that should NOT be run --- useful in conjunction 

28 with glob patterns. If a file is listed as "@fileName", the @ is 

29 stripped and we don't bother to check if fileName exists (useful for 

30 machine-generated files). 

31 expectedFailures : `dict`, optional 

32 A dictionary; the keys are tests that are known to fail; the values 

33 are strings to print. 

34 args : `dict`, optional 

35 A dictionary with testnames as keys, and argument strings as values. 

36 As scons always runs from the top-level directory, tests has to fiddle 

37 with paths. If an argument is a file this is done automatically; if 

38 it's e.g., just a basename then you have to tell tests that it's 

39 really (part of a) filename by prefixing the name by ``file:``. 

40 tmpDir : `str`, optional 

41 The location of the test outputs. 

42 verbose : `bool`, optional 

43 How chatty you want the test code to be. 

44 

45 Notes 

46 ----- 

47 Sample usage: 

48 

49 .. code-block:: python 

50 

51 tests = lsst.tests.Control( 

52 env, 

53 args={ 

54 "MaskIO_1" : "data/871034p_1_MI_msk.fits", 

55 "MaskedImage_1" : "file:data/871034p_1_MI foo", 

56 }, 

57 ignoreList=["Measure_1"], 

58 expectedFailures={"BBox_1": "Problem with single-pixel BBox"} 

59 ) 

60 

61 This class is unchanged from previous versions of sconsUtils, but it will 

62 now generally be called via 

63 `lsst.sconsUtils.scripts.BasicSConscript.tests`. 

64 """ 

65 

66 _IGNORE = "IGNORE" 

67 _EXPECT_FAILURE = "EXPECT_FAILURE" 

68 

69 def __init__(self, env, ignoreList=None, expectedFailures=None, args=None, 

70 tmpDir=".tests", verbose=False): 

71 

72 # Need to define our own Astropy cache directories. 

73 # Unfortunately we can not simply set XDG_CACHE_HOME 

74 # to $HOME/.astropy. Do not forward $HOME or the XDG_CONFIG_HOME 

75 # environment variables since those may affect test outcomes. 

76 xdgCacheVar = "XDG_CACHE_HOME" 

77 if xdgCacheVar not in os.environ: 

78 if "~" in os.path.expanduser("~"): 

79 state.log.warn(f"Neither $HOME nor ${xdgCacheVar} defined. No Astropy cache enabled.") 

80 else: 

81 # We need a directory for the cache and that directory 

82 # has to have an "astropy" directory inside it. We can 

83 # use ~/.astropy or ~/.lsst or a tmp directory but choose 

84 # ~/.lsst initially. 

85 cacheDir = os.path.expanduser("~/.lsst") 

86 astropyCacheDir = os.path.join(cacheDir, "astropy") 

87 if not os.path.exists(astropyCacheDir): 

88 os.makedirs(astropyCacheDir, exist_ok=True) # Race condition is okay 

89 os.environ[xdgCacheVar] = cacheDir 

90 else: 

91 if not os.path.exists(os.path.expanduser(os.path.join(os.environ[xdgCacheVar], 

92 "astropy"))): 

93 state.log.warn(f"{xdgCacheVar} is set but will not be used for " 

94 "astropy due to lack of astropy directory within it") 

95 

96 # Forward some environment to the tests 

97 for envvar in ["PYTHONPATH", "HTTP_PROXY", "HTTPS_PROXY", xdgCacheVar]: 

98 if envvar in os.environ: 

99 env.AppendENVPath(envvar, os.environ[envvar]) 

100 

101 # We can't use the AppendENVPath call above as that mangles anything 

102 # with ':' separators. 

103 if "PYTHONWARNINGS" in os.environ: 

104 env["ENV"]["PYTHONWARNINGS"] = os.environ["PYTHONWARNINGS"] 

105 

106 self._env = env 

107 

108 self._tmpDir = tmpDir 

109 self._cwd = os.path.abspath(os.path.curdir) 

110 

111 # Calculate the absolute path for temp dir if it is relative. 

112 # This assumes the temp dir is relative to where the tests SConscript 

113 # file is located. SCons will know how to handle this itself but 

114 # some options require the code to know where to write things. 

115 if os.path.isabs(self._tmpDir): 

116 self._tmpDirAbs = self._tmpDir 

117 else: 

118 self._tmpDirAbs = os.path.join(self._cwd, self._tmpDir) 

119 

120 self._verbose = verbose 

121 

122 self._info = {} # information about processing targets 

123 if ignoreList: 

124 for f in ignoreList: 

125 if f.startswith("@"): # @dfilename => don't complain if filename doesn't exist 

126 f = f[1:] 

127 else: 

128 if not os.path.exists(f): 

129 state.log.warn("You're ignoring a non-existent file, %s" % f) 

130 self._info[f] = (self._IGNORE, None) 

131 

132 if expectedFailures: 

133 for f in expectedFailures: 

134 self._info[f] = (self._EXPECT_FAILURE, expectedFailures[f]) 

135 

136 if args: 

137 self._args = args # arguments for tests 

138 else: 

139 self._args = {} 

140 

141 self.runExamples = True # should I run the examples? 

142 try: 

143 # file is user read/write/executable 

144 self.runExamples = (os.stat(self._tmpDir).st_mode & 0o700) != 0 

145 except OSError: 

146 pass 

147 

148 if not self.runExamples: 

149 print("Not running examples; \"chmod 755 %s\" to run them again" % self._tmpDir, 

150 file=sys.stderr) 

151 

152 def args(self, test): 

153 """Arguments to use for this test. 

154 

155 Parameters 

156 ---------- 

157 test : `str` 

158 Test file to be run. 

159 

160 Returns 

161 ------- 

162 args : `str` 

163 The arguments as a single string. An empty string is returned 

164 if no arguments were specified in the constructor. 

165 """ 

166 try: 

167 return self._args[test] 

168 except KeyError: 

169 return "" 

170 

171 def ignore(self, test): 

172 """Should the test be ignored. 

173 

174 Parameters 

175 ---------- 

176 test : `str` 

177 The test target name. 

178 

179 Returns 

180 ------- 

181 ignore : `bool` 

182 Whether the test should be ignored or not. 

183 """ 

184 if not test.endswith(".py") and \ 

185 len(self._env.Glob(test)) == 0: # we don't know how to build it 

186 return True 

187 

188 ignoreFile = test in self._info and self._info[test][0] == self._IGNORE 

189 

190 if self._verbose and ignoreFile: 

191 print("Skipping", test, file=sys.stderr) 

192 

193 return ignoreFile 

194 

195 def messages(self, test): 

196 """Return the messages to be used in case of success/failure. 

197 

198 Parameters 

199 ---------- 

200 test : `str` 

201 The test target. 

202 

203 Returns 

204 ------- 

205 messages : `tuple` 

206 A `tuple` containing four strings: whether the test should pass 

207 (as a value "true" or "false") and the associated message, and 

208 whether the test should fail and the associated message. 

209 """ 

210 

211 if test in self._info and self._info[test][0] == self._EXPECT_FAILURE: 

212 msg = self._info[test][1] 

213 return ("false", "Passed, but should have failed: %s" % msg, 

214 "true", "Failed as expected: %s" % msg) 

215 else: 

216 return ("true", "passed", 

217 "false", "failed") 

218 

219 def run(self, fileGlob): 

220 """Create a test target for each file matching the supplied glob. 

221 

222 Parameters 

223 ---------- 

224 fileGlob : `str` or `SCons.Environment.Glob` 

225 File matching glob. 

226 

227 Returns 

228 ------- 

229 targets : 

230 Test target for each matching file. 

231 """ 

232 

233 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node 

234 fileGlob = str(fileGlob) 

235 targets = [] 

236 if not self.runExamples: 

237 return targets 

238 

239 # Determine any library load path values that we have to prepend 

240 # to the command. 

241 libpathstr = utils.libraryLoaderEnvironment() 

242 

243 for f in glob.glob(fileGlob): 

244 interpreter = "" # interpreter to run test, if needed 

245 

246 if f.endswith(".cc"): # look for executable 

247 f = os.path.splitext(f)[0] 

248 else: 

249 interpreter = "pytest -Wd --durations=5 --junit-xml=${TARGET}.xml" 

250 interpreter += " --junit-prefix={0}".format(self.junitPrefix()) 

251 interpreter += " --log-level=DEBUG" 

252 interpreter += self._getPytestCoverageCommand() 

253 

254 if self.ignore(f): 

255 continue 

256 

257 target = os.path.join(self._tmpDir, f) 

258 

259 args = [] 

260 for a in self.args(f).split(" "): 

261 # if a is a file, make it an absolute name as scons runs from 

262 # the root directory 

263 filePrefix = "file:" 

264 if a.startswith(filePrefix): # they explicitly said that this was a file 

265 a = os.path.join(self._cwd, a[len(filePrefix):]) 

266 else: 

267 try: # see if it's a file 

268 os.stat(a) 

269 a = os.path.join(self._cwd, a) 

270 except OSError: 

271 pass 

272 

273 args += [a] 

274 

275 (should_pass, passedMsg, should_fail, failedMsg) = self.messages(f) 

276 

277 expandedArgs = " ".join(args) 

278 result = self._env.Command(target, f, """ 

279 @rm -f ${TARGET}.failed; 

280 @printf "%%s" 'running ${SOURCES}... '; 

281 @echo $SOURCES %s > $TARGET; echo >> $TARGET; 

282 @if %s %s $SOURCES %s >> $TARGET 2>&1; then \ 

283 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \ 

284 echo "%s"; \ 

285 else \ 

286 if ! %s; then mv $TARGET ${TARGET}.failed; fi; \ 

287 echo "%s"; \ 

288 fi; 

289 """ % (expandedArgs, libpathstr, interpreter, expandedArgs, should_pass, 

290 passedMsg, should_fail, failedMsg)) 

291 

292 targets.extend(result) 

293 

294 self._env.Alias(os.path.basename(target), target) 

295 

296 self._env.Clean(target, self._tmpDir) 

297 

298 return targets 

299 

300 def runPythonTests(self, pyList): 

301 """Add a single target for testing all python files. 

302 

303 Parameters 

304 ---------- 

305 pyList : `list` 

306 A list of nodes corresponding to python test files. The 

307 IgnoreList is respected when scanning for entries. If pyList 

308 is `None`, or an empty list, it uses automated test discovery 

309 within pytest. This differs from the behavior of 

310 `lsst.sconsUtils.BasicSconscript.tests` 

311 where a distinction is made. 

312 

313 Returns 

314 ------- 

315 target : `list` 

316 Returns a list containing a single target. 

317 """ 

318 

319 if pyList is None: 

320 pyList = [] 

321 

322 # Determine any library load path values that we have to prepend 

323 # to the command. 

324 libpathstr = utils.libraryLoaderEnvironment() 

325 

326 # Get list of python files with the path included. 

327 pythonTestFiles = [] 

328 for fileGlob in pyList: 

329 if not isinstance(fileGlob, str): # env.Glob() returns an scons Node 

330 fileGlob = str(fileGlob) 

331 for f in glob.glob(fileGlob): 

332 if self.ignore(f): 

333 continue 

334 pythonTestFiles.append(os.path.join(self._cwd, f)) 

335 

336 # Now set up the python testing target 

337 # We always want to run this with the tests target. 

338 # We have decided to use pytest caching so that on reruns we only 

339 # run failed tests. 

340 lfnfOpt = "none" if 'install' in SCons.Script.COMMAND_LINE_TARGETS else "all" 

341 interpreter = f"pytest -Wd --lf --lfnf={lfnfOpt}" 

342 interpreter += " --durations=5 --junit-xml=${TARGET} --session2file=${TARGET}.out" 

343 interpreter += " --junit-prefix={0}".format(self.junitPrefix()) 

344 interpreter += " --log-level=DEBUG" 

345 interpreter += self._getPytestCoverageCommand() 

346 

347 # Ignore doxygen build directories since they can confuse pytest 

348 # test collection 

349 interpreter += " --ignore=doc/html --ignore=doc/xml" 

350 

351 # Ignore the C++ directories since they will never have python 

352 # code and doing this will speed up test collection 

353 interpreter += " --ignore=src --ignore=include --ignore=lib" 

354 

355 # Ignore the eups directory 

356 interpreter += " --ignore=ups" 

357 

358 # We currently have a race condition in test collection when 

359 # examples has C++ code. Removing it from the scan will get us through 

360 # until we can fix the problem properly. Rely on GitHub PRs to 

361 # do the flake8 check. 

362 interpreter += " --ignore=examples" 

363 

364 # Also include temporary files made by compilers. 

365 # These can come from examples directories that include C++. 

366 interpreter += " --ignore-glob='*.tmp'" 

367 

368 target = os.path.join(self._tmpDir, "pytest-{}.xml".format(self._env['eupsProduct'])) 

369 

370 # Work out how many jobs scons has been configured to use 

371 # and use that number with pytest. This could cause trouble 

372 # if there are lots of binary tests to run and lots of singles. 

373 njobs = self._env.GetOption("num_jobs") 

374 print("Running pytest with {} process{}".format(njobs, "" if njobs == 1 else "es")) 

375 if njobs > 1: 

376 # We unambiguously specify the Python interpreter to be used to 

377 # execute tests. This ensures that all pytest-xdist worker 

378 # processes refer to the same Python as the xdist controller, and 

379 # hence avoids pollution of ``sys.path`` that can happen when we 

380 # call the same interpreter by different paths (for example, if 

381 # the controller process calls ``miniconda/bin/python``, and the 

382 # workers call ``current/bin/python``, the workers will end up 

383 # with site-packages directories corresponding to both locations 

384 # on ``sys.path``, even if the one is a symlink to the other). 

385 executable = os.path.realpath(sys.executable) 

386 

387 # if there is a space in the executable path we have to use the 

388 # original method and hope things work okay. This will be rare but 

389 # without this a space in the path is impossible because of how 

390 # xdist currently parses the tx option 

391 interpreter = interpreter + " --max-worker-restart=0" 

392 if " " not in executable: 

393 interpreter = (interpreter 

394 + " -d --tx={}*popen//python={}".format(njobs, executable)) 

395 else: 

396 interpreter = interpreter + " -n {}".format(njobs) 

397 

398 # Remove target so that we always trigger pytest 

399 if os.path.exists(target): 

400 os.unlink(target) 

401 

402 if not pythonTestFiles: 

403 print("pytest: automated test discovery mode enabled.") 

404 else: 

405 nfiles = len(pythonTestFiles) 

406 print("pytest: running on {} Python test file{}.".format(nfiles, "" if nfiles == 1 else "s")) 

407 

408 # If we ran all the test, then copy the previous test 

409 # execution products to `.all' files so we can retrieve later. 

410 # If we skip the test (exit code 5), retrieve those `.all' files. 

411 cmd = "" 

412 if lfnfOpt == "all": 

413 cmd += "@rm -f ${{TARGET}} ${{TARGET}}.failed;" 

414 cmd += """ 

415 @printf "%s\\n" 'running global pytest... '; 

416 @({2} {0} {1}); \ 

417 export rc="$?"; \ 

418 if [ "$$rc" -eq 0 ]; then \ 

419 echo "Global pytest run completed successfully"; \ 

420 cp ${{TARGET}} ${{TARGET}}.all || true; \ 

421 cp ${{TARGET}}.out ${{TARGET}}.out.all || true; \ 

422 elif [ "$$rc" -eq 5 ]; then \ 

423 echo "Global pytest run completed successfully - no tests ran"; \ 

424 mv ${{TARGET}}.all ${{TARGET}} || true; \ 

425 mv ${{TARGET}}.out.all ${{TARGET}}.out || true; \ 

426 else \ 

427 echo "Global pytest run: failed with $$rc"; \ 

428 mv ${{TARGET}}.out ${{TARGET}}.failed; \ 

429 fi; 

430 """ 

431 testfiles = " ".join([pipes.quote(p) for p in pythonTestFiles]) 

432 result = self._env.Command(target, None, cmd.format(interpreter, testfiles, libpathstr)) 

433 

434 self._env.Alias(os.path.basename(target), target) 

435 self._env.Clean(target, self._tmpDir) 

436 

437 return [result] 

438 

439 def junitPrefix(self): 

440 """Calculate the prefix to use for the JUnit output. 

441 

442 Returns 

443 ------- 

444 prefix : `str` 

445 Prefix string to use. 

446 

447 Notes 

448 ----- 

449 Will use the EUPS product being built and the value of the 

450 ``LSST_JUNIT_PREFIX`` environment variable if that is set. 

451 """ 

452 controlVar = "LSST_JUNIT_PREFIX" 

453 prefix = self._env['eupsProduct'] 

454 

455 if controlVar in os.environ: 

456 prefix += ".{0}".format(os.environ[controlVar]) 

457 

458 return prefix 

459 

460 def _getPytestCoverageCommand(self): 

461 """Form the additional arguments required to enable coverage testing. 

462 

463 Coverage output files are written using ``${TARGET}`` as a base. 

464 

465 Returns 

466 ------- 

467 options : `str` 

468 String defining the coverage-specific arguments to give to the 

469 pytest command. 

470 """ 

471 

472 options = "" 

473 

474 # Basis for deriving file names 

475 # We use the magic target from SCons. 

476 prefix = "${TARGET}" 

477 

478 # Only report coverage for files in the build tree. 

479 # If --cov is used full coverage will be reported for all installed 

480 # code as well, but that is probably a distraction as for this 

481 # test run we are only interested in coverage of this package. 

482 # Use "python" instead of "." to remove test files from coverage. 

483 options += " --cov=." 

484 

485 # Always enabled branch coverage and terminal summary 

486 options += " --cov-branch --cov-report=term " 

487 

488 covfile = "{}-cov-{}.xml".format(prefix, self._env['eupsProduct']) 

489 

490 # We should specify the output directory explicitly unless the prefix 

491 # indicates that we are using the SCons target 

492 if covfile.startswith("${TARGET}"): 

493 covpath = covfile 

494 else: 

495 covpath = os.path.join(self._tmpDirAbs, covfile) 

496 options += " --cov-report=xml:'{}'".format(covpath) 

497 

498 # Use the prefix for the HTML output directory 

499 htmlfile = ":'{}-htmlcov'".format(prefix) 

500 options += " --cov-report=html{}".format(htmlfile) 

501 

502 return options