Coverage for tests/test_executors.py: 21%

Shortcuts on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

341 statements  

1# This file is part of ctrl_mpexec. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22"""Simple unit test for cmdLineFwk module. 

23""" 

24 

25import logging 

26import signal 

27import sys 

28import time 

29import unittest 

30import warnings 

31from multiprocessing import Manager 

32 

33import networkx as nx 

34import psutil 

35from lsst.ctrl.mpexec import ( 

36 ExecutionStatus, 

37 MPGraphExecutor, 

38 MPGraphExecutorError, 

39 MPTimeoutError, 

40 QuantumExecutor, 

41 QuantumReport, 

42) 

43from lsst.ctrl.mpexec.execFixupDataId import ExecFixupDataId 

44from lsst.pipe.base import NodeId 

45 

46logging.basicConfig(level=logging.DEBUG) 

47 

48_LOG = logging.getLogger(__name__) 

49 

50 

51class QuantumExecutorMock(QuantumExecutor): 

52 """Mock class for QuantumExecutor""" 

53 

54 def __init__(self, mp=False): 

55 self.quanta = [] 

56 if mp: 

57 # in multiprocess mode use shared list 

58 manager = Manager() 

59 self.quanta = manager.list() 

60 self.report = None 

61 self._execute_called = False 

62 

63 def execute(self, taskDef, quantum, butler): 

64 _LOG.debug("QuantumExecutorMock.execute: taskDef=%s dataId=%s", taskDef, quantum.dataId) 

65 self._execute_called = True 

66 if taskDef.taskClass: 

67 try: 

68 # only works for one of the TaskMock classes below 

69 taskDef.taskClass().runQuantum() 

70 self.report = QuantumReport(dataId=quantum.dataId, taskLabel=taskDef.label) 

71 except Exception as exc: 

72 self.report = QuantumReport.from_exception( 

73 exception=exc, 

74 dataId=quantum.dataId, 

75 taskLabel=taskDef.label, 

76 ) 

77 raise 

78 self.quanta.append(quantum) 

79 return quantum 

80 

81 def getReport(self): 

82 if not self._execute_called: 

83 raise RuntimeError("getReport called before execute") 

84 return self.report 

85 

86 def getDataIds(self, field): 

87 """Returns values for dataId field for each visited quanta""" 

88 return [quantum.dataId[field] for quantum in self.quanta] 

89 

90 

91class QuantumMock: 

92 def __init__(self, dataId): 

93 self.dataId = dataId 

94 

95 def __eq__(self, other): 

96 return self.dataId == other.dataId 

97 

98 def __hash__(self): 

99 # dict.__eq__ is order-insensitive 

100 return hash(tuple(sorted(kv for kv in self.dataId.items()))) 

101 

102 

103class QuantumIterDataMock: 

104 """Simple class to mock QuantumIterData.""" 

105 

106 def __init__(self, index, taskDef, **dataId): 

107 self.index = index 

108 self.taskDef = taskDef 

109 self.quantum = QuantumMock(dataId) 

110 self.dependencies = set() 

111 self.nodeId = NodeId(index, "DummyBuildString") 

112 

113 

114class QuantumGraphMock: 

115 """Mock for quantum graph.""" 

116 

117 def __init__(self, qdata): 

118 self._graph = nx.DiGraph() 

119 previous = qdata[0] 

120 for node in qdata[1:]: 

121 self._graph.add_edge(previous, node) 

122 previous = node 

123 

124 def __iter__(self): 

125 yield from nx.topological_sort(self._graph) 

126 

127 def __len__(self): 

128 return len(self._graph) 

129 

130 def findTaskDefByLabel(self, label): 

131 for q in self: 

132 if q.taskDef.label == label: 

133 return q.taskDef 

134 

135 def getQuantaForTask(self, taskDef): 

136 nodes = self.getNodesForTask(taskDef) 

137 return {q.quantum for q in nodes} 

138 

139 def getNodesForTask(self, taskDef): 

140 quanta = set() 

141 for q in self: 

142 if q.taskDef == taskDef: 

143 quanta.add(q) 

144 return quanta 

145 

146 @property 

147 def graph(self): 

148 return self._graph 

149 

150 def findCycle(self): 

151 return [] 

152 

153 def determineInputsToQuantumNode(self, node): 

154 result = set() 

155 for n in node.dependencies: 

156 for otherNode in self: 

157 if otherNode.index == n: 

158 result.add(otherNode) 

159 return result 

160 

161 

162class TaskMockMP: 

163 """Simple mock class for task supporting multiprocessing.""" 

164 

165 canMultiprocess = True 

166 

167 def runQuantum(self): 

168 _LOG.debug("TaskMockMP.runQuantum") 

169 pass 

170 

171 

172class TaskMockFail: 

173 """Simple mock class for task which fails.""" 

174 

175 canMultiprocess = True 

176 

177 def runQuantum(self): 

178 _LOG.debug("TaskMockFail.runQuantum") 

179 raise ValueError("expected failure") 

180 

181 

182class TaskMockCrash: 

183 """Simple mock class for task which fails.""" 

184 

185 canMultiprocess = True 

186 

187 def runQuantum(self): 

188 _LOG.debug("TaskMockCrash.runQuantum") 

189 signal.raise_signal(signal.SIGILL) 

190 

191 

192class TaskMockSleep: 

193 """Simple mock class for task which "runs" for some time.""" 

194 

195 canMultiprocess = True 

196 

197 def runQuantum(self): 

198 _LOG.debug("TaskMockSleep.runQuantum") 

199 time.sleep(5.0) 

200 

201 

202class TaskMockLongSleep: 

203 """Simple mock class for task which "runs" for very long time.""" 

204 

205 canMultiprocess = True 

206 

207 def runQuantum(self): 

208 _LOG.debug("TaskMockLongSleep.runQuantum") 

209 time.sleep(100.0) 

210 

211 

212class TaskMockNoMP: 

213 """Simple mock class for task not supporting multiprocessing.""" 

214 

215 canMultiprocess = False 

216 

217 

218class TaskDefMock: 

219 """Simple mock class for task definition in a pipeline.""" 

220 

221 def __init__(self, taskName="Task", config=None, taskClass=TaskMockMP, label="task1"): 

222 self.taskName = taskName 

223 self.config = config 

224 self.taskClass = taskClass 

225 self.label = label 

226 

227 def __str__(self): 

228 return f"TaskDefMock(taskName={self.taskName}, taskClass={self.taskClass.__name__})" 

229 

230 

231def _count_status(report, status): 

232 """Count number of quanta witha a given status.""" 

233 return len([qrep for qrep in report.quantaReports if qrep.status is status]) 

234 

235 

236class MPGraphExecutorTestCase(unittest.TestCase): 

237 """A test case for MPGraphExecutor class""" 

238 

239 def test_mpexec_nomp(self): 

240 """Make simple graph and execute""" 

241 

242 taskDef = TaskDefMock() 

243 qgraph = QuantumGraphMock( 

244 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

245 ) 

246 

247 # run in single-process mode 

248 qexec = QuantumExecutorMock() 

249 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

250 mpexec.execute(qgraph, butler=None) 

251 self.assertEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

252 report = mpexec.getReport() 

253 self.assertEqual(report.status, ExecutionStatus.SUCCESS) 

254 self.assertIsNone(report.exitCode) 

255 self.assertIsNone(report.exceptionInfo) 

256 self.assertEqual(len(report.quantaReports), 3) 

257 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports)) 

258 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports)) 

259 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

260 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports)) 

261 

262 def test_mpexec_mp(self): 

263 """Make simple graph and execute""" 

264 

265 taskDef = TaskDefMock() 

266 qgraph = QuantumGraphMock( 

267 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

268 ) 

269 

270 methods = ["spawn"] 

271 if sys.platform == "linux": 

272 methods.append("fork") 

273 methods.append("forkserver") 

274 

275 for method in methods: 

276 with self.subTest(startMethod=method): 

277 # Run in multi-process mode, the order of results is not 

278 # defined. 

279 qexec = QuantumExecutorMock(mp=True) 

280 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, startMethod=method) 

281 mpexec.execute(qgraph, butler=None) 

282 self.assertCountEqual(qexec.getDataIds("detector"), [0, 1, 2]) 

283 report = mpexec.getReport() 

284 self.assertEqual(report.status, ExecutionStatus.SUCCESS) 

285 self.assertIsNone(report.exitCode) 

286 self.assertIsNone(report.exceptionInfo) 

287 self.assertEqual(len(report.quantaReports), 3) 

288 self.assertTrue(all(qrep.status == ExecutionStatus.SUCCESS for qrep in report.quantaReports)) 

289 self.assertTrue(all(qrep.exitCode == 0 for qrep in report.quantaReports)) 

290 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

291 self.assertTrue(all(qrep.taskLabel == "task1" for qrep in report.quantaReports)) 

292 

293 def test_mpexec_nompsupport(self): 

294 """Try to run MP for task that has no MP support which should fail""" 

295 

296 taskDef = TaskDefMock(taskClass=TaskMockNoMP) 

297 qgraph = QuantumGraphMock( 

298 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

299 ) 

300 

301 # run in multi-process mode 

302 qexec = QuantumExecutorMock() 

303 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

304 with self.assertRaisesRegex(MPGraphExecutorError, "Task Task does not support multiprocessing"): 

305 mpexec.execute(qgraph, butler=None) 

306 

307 def test_mpexec_fixup(self): 

308 """Make simple graph and execute, add dependencies by executing fixup 

309 code. 

310 """ 

311 

312 taskDef = TaskDefMock() 

313 

314 for reverse in (False, True): 

315 qgraph = QuantumGraphMock( 

316 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(3)] 

317 ) 

318 

319 qexec = QuantumExecutorMock() 

320 fixup = ExecFixupDataId("task1", "detector", reverse=reverse) 

321 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec, executionGraphFixup=fixup) 

322 mpexec.execute(qgraph, butler=None) 

323 

324 expected = [0, 1, 2] 

325 if reverse: 

326 expected = list(reversed(expected)) 

327 self.assertEqual(qexec.getDataIds("detector"), expected) 

328 

329 def test_mpexec_timeout(self): 

330 """Fail due to timeout""" 

331 

332 taskDef = TaskDefMock() 

333 taskDefSleep = TaskDefMock(taskClass=TaskMockSleep) 

334 qgraph = QuantumGraphMock( 

335 [ 

336 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

337 QuantumIterDataMock(index=1, taskDef=taskDefSleep, detector=1), 

338 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

339 ] 

340 ) 

341 

342 # with failFast we'll get immediate MPTimeoutError 

343 qexec = QuantumExecutorMock(mp=True) 

344 mpexec = MPGraphExecutor(numProc=3, timeout=1, quantumExecutor=qexec, failFast=True) 

345 with self.assertRaises(MPTimeoutError): 

346 mpexec.execute(qgraph, butler=None) 

347 report = mpexec.getReport() 

348 self.assertEqual(report.status, ExecutionStatus.TIMEOUT) 

349 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError") 

350 self.assertGreater(len(report.quantaReports), 0) 

351 self.assertEquals(_count_status(report, ExecutionStatus.TIMEOUT), 1) 

352 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports)) 

353 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

354 

355 # with failFast=False exception happens after last task finishes 

356 qexec = QuantumExecutorMock(mp=True) 

357 mpexec = MPGraphExecutor(numProc=3, timeout=3, quantumExecutor=qexec, failFast=False) 

358 with self.assertRaises(MPTimeoutError): 

359 mpexec.execute(qgraph, butler=None) 

360 # We expect two tasks (0 and 2) to finish successfully and one task to 

361 # timeout. Unfortunately on busy CPU there is no guarantee that tasks 

362 # finish on time, so expect more timeouts and issue a warning. 

363 detectorIds = set(qexec.getDataIds("detector")) 

364 self.assertLess(len(detectorIds), 3) 

365 if detectorIds != {0, 2}: 

366 warnings.warn(f"Possibly timed out tasks, expected [0, 2], received {detectorIds}") 

367 report = mpexec.getReport() 

368 self.assertEqual(report.status, ExecutionStatus.TIMEOUT) 

369 self.assertEqual(report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPTimeoutError") 

370 self.assertGreater(len(report.quantaReports), 0) 

371 self.assertGreater(_count_status(report, ExecutionStatus.TIMEOUT), 0) 

372 self.assertTrue(any(qrep.exitCode < 0 for qrep in report.quantaReports)) 

373 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

374 

375 def test_mpexec_failure(self): 

376 """Failure in one task should not stop other tasks""" 

377 

378 taskDef = TaskDefMock() 

379 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

380 qgraph = QuantumGraphMock( 

381 [ 

382 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

383 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

384 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

385 ] 

386 ) 

387 

388 qexec = QuantumExecutorMock(mp=True) 

389 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

390 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

391 mpexec.execute(qgraph, butler=None) 

392 self.assertCountEqual(qexec.getDataIds("detector"), [0, 2]) 

393 report = mpexec.getReport() 

394 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

395 self.assertEqual( 

396 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

397 ) 

398 self.assertGreater(len(report.quantaReports), 0) 

399 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

400 self.assertEquals(_count_status(report, ExecutionStatus.SUCCESS), 2) 

401 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

402 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

403 

404 def test_mpexec_failure_dep(self): 

405 """Failure in one task should skip dependents""" 

406 

407 taskDef = TaskDefMock() 

408 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

409 qdata = [ 

410 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

411 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

412 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

413 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

414 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

415 ] 

416 qdata[2].dependencies.add(1) 

417 qdata[4].dependencies.add(3) 

418 qdata[4].dependencies.add(2) 

419 

420 qgraph = QuantumGraphMock(qdata) 

421 

422 qexec = QuantumExecutorMock(mp=True) 

423 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

424 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

425 mpexec.execute(qgraph, butler=None) 

426 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

427 report = mpexec.getReport() 

428 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

429 self.assertEqual( 

430 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

431 ) 

432 # Dependencies of failed tasks do not appear in quantaReports 

433 self.assertGreater(len(report.quantaReports), 0) 

434 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

435 self.assertEquals(_count_status(report, ExecutionStatus.SUCCESS), 2) 

436 self.assertEquals(_count_status(report, ExecutionStatus.SKIPPED), 2) 

437 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

438 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

439 

440 def test_mpexec_failure_dep_nomp(self): 

441 """Failure in one task should skip dependents, in-process version""" 

442 

443 taskDef = TaskDefMock() 

444 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

445 qdata = [ 

446 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

447 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

448 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

449 QuantumIterDataMock(index=3, taskDef=taskDef, detector=3), 

450 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

451 ] 

452 qdata[2].dependencies.add(1) 

453 qdata[4].dependencies.add(3) 

454 qdata[4].dependencies.add(2) 

455 

456 qgraph = QuantumGraphMock(qdata) 

457 

458 qexec = QuantumExecutorMock() 

459 mpexec = MPGraphExecutor(numProc=1, timeout=100, quantumExecutor=qexec) 

460 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

461 mpexec.execute(qgraph, butler=None) 

462 self.assertCountEqual(qexec.getDataIds("detector"), [0, 3]) 

463 report = mpexec.getReport() 

464 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

465 self.assertEqual( 

466 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

467 ) 

468 # Dependencies of failed tasks do not appear in quantaReports 

469 self.assertGreater(len(report.quantaReports), 0) 

470 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

471 self.assertEquals(_count_status(report, ExecutionStatus.SUCCESS), 2) 

472 self.assertEquals(_count_status(report, ExecutionStatus.SKIPPED), 2) 

473 self.assertTrue(all(qrep.exitCode is None for qrep in report.quantaReports)) 

474 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

475 

476 def test_mpexec_failure_failfast(self): 

477 """Fast fail stops quickly. 

478 

479 Timing delay of task #3 should be sufficient to process 

480 failure and raise exception. 

481 """ 

482 

483 taskDef = TaskDefMock() 

484 taskDefFail = TaskDefMock(taskClass=TaskMockFail) 

485 taskDefLongSleep = TaskDefMock(taskClass=TaskMockLongSleep) 

486 qdata = [ 

487 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

488 QuantumIterDataMock(index=1, taskDef=taskDefFail, detector=1), 

489 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

490 QuantumIterDataMock(index=3, taskDef=taskDefLongSleep, detector=3), 

491 QuantumIterDataMock(index=4, taskDef=taskDef, detector=4), 

492 ] 

493 qdata[1].dependencies.add(0) 

494 qdata[2].dependencies.add(1) 

495 qdata[4].dependencies.add(3) 

496 qdata[4].dependencies.add(2) 

497 

498 qgraph = QuantumGraphMock(qdata) 

499 

500 qexec = QuantumExecutorMock(mp=True) 

501 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

502 with self.assertRaisesRegex(MPGraphExecutorError, "failed, exit code=1"): 

503 mpexec.execute(qgraph, butler=None) 

504 self.assertCountEqual(qexec.getDataIds("detector"), [0]) 

505 report = mpexec.getReport() 

506 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

507 self.assertEqual( 

508 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

509 ) 

510 # Dependencies of failed tasks do not appear in quantaReports 

511 self.assertGreater(len(report.quantaReports), 0) 

512 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

513 self.assertTrue(any(qrep.exitCode > 0 for qrep in report.quantaReports)) 

514 self.assertTrue(any(qrep.exceptionInfo is not None for qrep in report.quantaReports)) 

515 

516 def test_mpexec_crash(self): 

517 """Check task crash due to signal""" 

518 

519 taskDef = TaskDefMock() 

520 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash) 

521 qgraph = QuantumGraphMock( 

522 [ 

523 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

524 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1), 

525 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

526 ] 

527 ) 

528 

529 qexec = QuantumExecutorMock(mp=True) 

530 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

531 with self.assertRaisesRegex(MPGraphExecutorError, "One or more tasks failed"): 

532 mpexec.execute(qgraph, butler=None) 

533 report = mpexec.getReport() 

534 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

535 self.assertEqual( 

536 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

537 ) 

538 # Dependencies of failed tasks do not appear in quantaReports 

539 self.assertGreater(len(report.quantaReports), 0) 

540 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

541 self.assertEquals(_count_status(report, ExecutionStatus.SUCCESS), 2) 

542 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports)) 

543 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

544 

545 def test_mpexec_crash_failfast(self): 

546 """Check task crash due to signal with --fail-fast""" 

547 

548 taskDef = TaskDefMock() 

549 taskDefCrash = TaskDefMock(taskClass=TaskMockCrash) 

550 qgraph = QuantumGraphMock( 

551 [ 

552 QuantumIterDataMock(index=0, taskDef=taskDef, detector=0), 

553 QuantumIterDataMock(index=1, taskDef=taskDefCrash, detector=1), 

554 QuantumIterDataMock(index=2, taskDef=taskDef, detector=2), 

555 ] 

556 ) 

557 

558 qexec = QuantumExecutorMock(mp=True) 

559 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec, failFast=True) 

560 with self.assertRaisesRegex(MPGraphExecutorError, "failed, killed by signal 4 .Illegal instruction"): 

561 mpexec.execute(qgraph, butler=None) 

562 report = mpexec.getReport() 

563 self.assertEqual(report.status, ExecutionStatus.FAILURE) 

564 self.assertEqual( 

565 report.exceptionInfo.className, "lsst.ctrl.mpexec.mpGraphExecutor.MPGraphExecutorError" 

566 ) 

567 self.assertEquals(_count_status(report, ExecutionStatus.FAILURE), 1) 

568 self.assertTrue(any(qrep.exitCode == -signal.SIGILL for qrep in report.quantaReports)) 

569 self.assertTrue(all(qrep.exceptionInfo is None for qrep in report.quantaReports)) 

570 

571 def test_mpexec_num_fd(self): 

572 """Check that number of open files stays reasonable""" 

573 

574 taskDef = TaskDefMock() 

575 qgraph = QuantumGraphMock( 

576 [QuantumIterDataMock(index=i, taskDef=taskDef, detector=i) for i in range(20)] 

577 ) 

578 

579 this_proc = psutil.Process() 

580 num_fds_0 = this_proc.num_fds() 

581 

582 # run in multi-process mode, the order of results is not defined 

583 qexec = QuantumExecutorMock(mp=True) 

584 mpexec = MPGraphExecutor(numProc=3, timeout=100, quantumExecutor=qexec) 

585 mpexec.execute(qgraph, butler=None) 

586 

587 num_fds_1 = this_proc.num_fds() 

588 # They should be the same but allow small growth just in case. 

589 # Without DM-26728 fix the difference would be equal to number of 

590 # quanta (20). 

591 self.assertLess(num_fds_1 - num_fds_0, 5) 

592 

593 

594if __name__ == "__main__": 594 ↛ 595line 594 didn't jump to line 595, because the condition on line 594 was never true

595 unittest.main()